source: trunk/MagicSoft/Mars/mjobs/MDataSet.cc@ 7740

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 1/2005 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2004-2005
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MDataSet
//
//  This class describes a collection of sequences.
//
//  Such an input file looks like:
//
//     crab.seq:
//     ---------
//       AnalysisNumber: 1
//
//       SequencesOn: 35222
//       SequencesOff: 36817
//
//       Sequence00035222.File: sequences/sequence035222.txt
//       Sequence00036817.File: sequences/sequence036817.txt
//
//       Sequence00035222.Dir: /data2/wuerzburg/Crab-Analyse/images/035222
//       Sequence00036817.Dir: /data2/wuerzburg/Crab-Analyse/images/036817
//
// The analysis number is an artifical number used to name the output
// files automatically if the names are not overwritten in the corresponding
// programs.
//
// The sequence number are used to concatenate the filenames of the
// sequences using the file structure used in the datacenter. Each sequence
// can be added to the on and off data at the same time but only once.
//
// If you have different file names you can overwrite the default file names
// using Sequence%08d.File (make sure you have 8 digits!)
//
// In standard coditions (datacenter file system) paths are concatenated
// by using the information in the sequence files (date, etc). You can
// overwrite the directories in which the sequence-files (eg I-files) are
// stored using Sequence%08d.Dir (make sure you have 8 digits!)
//
// Resource file entries are case sensitive!
//
// IMPORTANT:
//   * Run filenames must begin with a string which allows correct
//     ordering in time, otherwise synchronization might fail.
//   * Sequence filenames should also have names allowing to order them
//     in time, but it is not necessary.
//
/////////////////////////////////////////////////////////////////////////////
#include "MDataSet.h"

#include <string.h>  // necessary for Fedora core 2 with kernel 2.6.9-1.667 #1 and gcc 3.4.2
#include <errno.h>   // necessary for Fedora core 2 with kernel 2.6.9-1.667 #1 and gcc 3.4.2

#include <stdlib.h>
#include <fstream>

#include <TEnv.h>
#include <TChain.h>
#include <TRegexp.h>
#include <TSystem.h> // TSystem::ExpandPath

#include "MLog.h"
#include "MLogManip.h"

#include "MRead.h"
#include "MJob.h"
#include "MAstro.h"
#include "MDirIter.h"
#include "MSequence.h"
#include "MPointingPos.h"

ClassImp(MDataSet);

using namespace std;

const TString MDataSet::fgCatalog       = "/magic/datacenter/setup/magic_favorites.edb";
const TString MDataSet::fgPathDataFiles = "/magic/data/star";
const TString MDataSet::fgPathSequences = "/magic/sequences";

// --------------------------------------------------------------------------
//
// Copy the sequence numbers from the TString runs into the TArrayI data
// Sequences which are twice in the list are only added once. In this case
// a warning is emitted.
//
void MDataSet::Split(TString &runs, TArrayI &data) const
{
    const TRegexp regexp("[0-9]+");

    data.Set(0);
    runs = runs.Strip(TString::kTrailing);

    while (!runs.IsNull())
    {
        const TString num = runs(regexp);

        if (num.IsNull())
        {
            *fLog << warn << "WARNING - Sequence is NaN (not a number): " << runs << endl;
            break;
        }

        const Int_t seq = atoi(num.Data());
        const Int_t n   = data.GetSize();

        // skip already existing entries
        int i;
        for (i=0; i<n; i++)
            if (data[i] == seq)
                break;

        if (i<n)
            *fLog << warn << "WARNING - Sequence #" << seq << " already in list... skipped." << endl;
        else
        {
            // set new entry
            data.Set(n+1);
            data[n] = seq;
        }

        // remove entry from string
        runs.Remove(0, runs.First(num)+num.Length());
    }

    MJob::SortArray(data);
}

// --------------------------------------------------------------------------
//
// After resolving the sequence filename and directory either from the
// default (/magic/data/sequences/0004/sequence00004000.txt) or from
// the corresponding entries in the dataset file.
// The entries are sorted by filename.
//
void MDataSet::ResolveSequences(TEnv &env, const TArrayI &num, TList &list, const TString &sequences, const TString &data) const
{
    for (int i=0; i<num.GetSize(); i++)
    {
        TString name = env.GetValue(Form("Sequence%08d.File", num[i]), "");
        TString dir  = env.GetValue(Form("Sequence%08d.Dir",  num[i]), "");

        // Set default sequence file and dir name
        if (name.IsNull())
            name = Form("%s%04d/sequence%08d.txt", sequences.Data(), num[i]/10000, num[i]);
        if (dir.IsNull())
            dir = Form("%s%04d/%08d", data.Data(), num[i]/10000, num[i]);

        gSystem->ExpandPathName(name);
        gSystem->ExpandPathName(dir);

        if (gSystem->AccessPathName(name, kFileExists))
            gLog << warn << "WARNING - Sequence file '" << name << "' doesn't exist." << endl;

        if (gSystem->AccessPathName(dir, kFileExists))
            gLog << warn << "WARNING - Directory '" << dir << "' doesn't exist." << endl;

        list.Add(new TNamed(name, dir));
    }

    // For the synchronization we must make sure, that all sequences are
    // in the correct order...
    // list.Sort();
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
MDataSet::MDataSet(const char *fname, TString sequences, TString data)
{
    fName  = fname;

    fSequencesOn.SetOwner();
    fSequencesOff.SetOwner();

    const char *expname = gSystem->ExpandPathName(fname);

    fTitle = Form("Dataset contained in file %s", expname);

    const Bool_t access = !gSystem->AccessPathName(expname, kFileExists);
    if (!access)
        gLog << err << "ERROR - Dataset file " << expname << " not accessible!" << endl;

    TEnv env(expname);
    delete [] expname;

    fNumAnalysis = env.GetValue("AnalysisNumber",  -1);

    TString str;
    str = env.GetValue("SequencesOn",  "");
    Split(str, fNumSequencesOn);
    str = env.GetValue("SequencesOff", "");
    Split(str, fNumSequencesOff);

    SetupDefaultPath(sequences, fgPathSequences);
    SetupDefaultPath(data,      fgPathDataFiles);

    ResolveSequences(env, fNumSequencesOn,  fSequencesOn,  sequences, data);
    ResolveSequences(env, fNumSequencesOff, fSequencesOff, sequences, data);

    fNameSource   = env.GetValue("SourceName", "");
    fCatalog      = env.GetValue("Catalog",    fgCatalog);
    fIsWobbleMode = env.GetValue("WobbleMode", kFALSE);
    fComment      = env.GetValue("Comment",    "");

    fNameSource = fNameSource.Strip(TString::kBoth);
    fCatalog    = fCatalog.Strip(TString::kBoth);
}

// --------------------------------------------------------------------------
//
// Return '+' if both can be accessed, '-' otherwise.
//
void MDataSet::PrintFile(const TObject &obj)
{
    const Char_t access =
        !gSystem->AccessPathName(obj.GetName(), kFileExists) &&
        !gSystem->AccessPathName(obj.GetTitle(), kFileExists) ? '+' : '-';

    gLog << " " << access << " " << obj.GetName() << " <" << obj.GetTitle() << ">" << endl;
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence
//
void MDataSet::Print(Option_t *o) const
{
    gLog << all;
    if (!IsValid())
    {
        gLog << "Dataset: " << fName << " <invalid - no analysis number available>" << endl;
        return;
    }
    gLog << "AnalysisNumber: " << fNumAnalysis << endl << endl;
    gLog << "SequencesOn:   ";
    for (int i=0; i<fNumSequencesOn.GetSize(); i++)
        gLog << " " << fNumSequencesOn[i];
    gLog << endl;
    gLog << "SequencesOff:  ";
    for (int i=0; i<fNumSequencesOff.GetSize(); i++)
        gLog << " " << fNumSequencesOff[i];
    gLog << endl << endl;

    gLog << "SourceName: " << fNameSource << endl;
    gLog << "Catalog: " << fCatalog << endl;

    gLog << "WobbleMode: " << (fIsWobbleMode?"On":"Off") << endl << endl;

    gLog << "Comment: " << fComment << endl;

    if (TString(o).Contains("files", TString::kIgnoreCase))
    {
        TObject *obj=0;

        gLog << endl;
        gLog << "On-Data Files:" << endl;
        TIter NextOn(&fSequencesOn);
        while ((obj=NextOn()))
            PrintFile(*obj);

        gLog << endl;
        gLog << "Off-Data Files:" << endl;
        TIter NextOff(&fSequencesOff);
        while ((obj=NextOff()))
            PrintFile(*obj);

        return;
    }
}

// --------------------------------------------------------------------------
//
// Adds all sequences contained in list to the MDirIter. After adding
// everything MDirIter::Sort is called to sort all entries by name.
//
Bool_t MDataSet::AddSequencesFromList(const TList &list, MDirIter &files)
{
    TIter Next(const_cast<TList*>(&list));
    TObject *o=0;
    while ((o=Next()))
    {
        MSequence seq(o->GetName());
        if (!seq.IsValid())
        {
            gLog << warn;
            gLog << "WARNING - MDataSet::AddSequencesFromList: Sequence invalid!" << endl;
            gLog << "  + File: " << o->GetName() << endl;
            gLog << "  + Dir:  " << o->GetTitle() << endl;
            return kFALSE;
        }

        const TString dir(o->GetTitle());
        seq.SetupDatRuns(files, MSequence::kImages, dir.IsNull() ? 0 : dir.Data());
    }

    // This is important in case of synchronisation, because the
    // files in the sequences can be interleaved (eg W1, W2)
    // Filenames MUST begin with an appropriate string which allow
    // to order them correctly in time!
    // files.Sort();

    if (gLog.GetDebugLevel()>4)
    {
        gLog << dbg << "Files which are searched:" << endl;
        files.Print();
    }
    return kTRUE;
}

Bool_t MDataSet::AddFilesOn(MRead &read) const
{
    MDirIter files;
    if (!AddSequencesFromList(fSequencesOn, files))
        return kFALSE;
    return read.AddFiles(files)>0;
}

Bool_t MDataSet::AddFilesOff(MRead &read) const
{
    MDirIter files;
    if (!AddSequencesFromList(fSequencesOff, files))
        return kFALSE;
    return read.AddFiles(files)>0;
}

Bool_t MDataSet::AddFiles(MRead &read) const
{
    const Bool_t rc1 = AddFilesOff(read);
    const Bool_t rc2 = AddFilesOn(read);
    return rc1 && rc2;
}

Int_t MDataSet::AddFilesToChain(MDirIter &files, TChain &chain)
{
    Int_t num=0;
    while (1)
    {
        const TString fname = files.Next();
        if (fname.IsNull())
            break;

        const Int_t n = chain.Add(fname);
        if (n<=0)
            return kFALSE;
        num += n;
    }
    return num;
}

Bool_t MDataSet::AddFilesOn(TChain &chain) const
{
    MDirIter files;
    if (!AddSequencesFromList(fSequencesOn, files))
        return kFALSE;
    return AddFilesToChain(files, chain)>0;
}

Bool_t MDataSet::AddFilesOff(TChain &chain) const
{
    MDirIter files;
    if (!AddSequencesFromList(fSequencesOff, files))
        return kFALSE;
    return AddFilesToChain(files, chain)>0;
}

Bool_t MDataSet::AddFiles(TChain &read) const
{
    const Bool_t rc1 = AddFilesOff(read);
    const Bool_t rc2 = AddFilesOn(read);
    return rc1 && rc2;
}

Bool_t MDataSet::GetSourcePos(MPointingPos &pos) const
{
    if (!HasSource())
    {
        gLog << err << "ERROR - MDataSet::GetSourcePos called, but no source available." << endl;
        return kFALSE;
    }

    TString catalog(fCatalog);
    gSystem->ExpandPathName(catalog);

    ifstream fin(catalog);
    if (!fin)
    {
        gLog << err << "Cannot open file " << catalog << ": ";
        gLog << strerror(errno) << endl;
        return kFALSE;
    }

    TString ra, dec, epoch;

    Int_t n = 0;
    while (1)
    {
        TString line;
        line.ReadLine(fin);
        if (!fin)
        {
            gLog << err << "ERROR - Source '" << fNameSource << "' not found in " << catalog << "." << endl;
            return kFALSE;
        }

        n++;

        TObjArray *arr = line.Tokenize(",");

        if (arr->GetEntries()<6)
        {
            gLog << err << "ERROR - Not enough arguments in line #" << n << " of " << catalog << endl;
            delete arr;
            return kFALSE;;
        }

        const TString name = (*arr)[0]->GetName();

        ra    = (*arr)[2]->GetName();
        dec   = (*arr)[3]->GetName();
        epoch = (*arr)[5]->GetName();

        delete arr;

        if (name.Strip(TString::kBoth)==fNameSource)
            break;
    }

    if (epoch.Strip(TString::kBoth)!=(TString)"2000")
    {
        gLog << err << "ERROR - Epoch not 2000... not supported." << endl;
        return kFALSE;
    }

    Double_t r,d;
    if (!MAstro::Coordinate2Angle(ra, r))
    {
        gLog << err << "ERROR - Interpreting right ascension: " << ra << endl;
        return kFALSE;
    }
    if (!MAstro::Coordinate2Angle(dec, d))
    {
        gLog << err << "ERROR - Interpreting declination: " << dec << endl;
        return kFALSE;
    }

    pos.SetSkyPosition(r, d);
    pos.SetTitle(fNameSource);

    return kTRUE;
}

// --------------------------------------------------------------------------
//
// Calls ReplaceAll(old, news) for all Dir-entries
//
void MDataSet::ReplaceDir(TList &list, const TString &old, const TString &news) const
{
    TIter Next(&list);
    TNamed *name = 0;
    while ((name=(TNamed*)Next()))
    {
        TString dir = name->GetTitle();
        dir.ReplaceAll(old, news);
        name->SetTitle(dir);
    }
}

// --------------------------------------------------------------------------
//
// Calls ReplaceAll(old, news) for all File-entries
//
void MDataSet::ReplaceFile(TList &list, const TString &old, const TString &news) const
{
    TIter Next(&list);
    TNamed *name = 0;
    while ((name=(TNamed*)Next()))
    {
        TString file = name->GetName();
        file.ReplaceAll(old, news);
        name->SetName(file);
    }
}