source: trunk/MagicSoft/Mars/mjobs/MDataSet.cc@ 6906

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 1/2005 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2004-2005
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MDataSet
//
//  This class describes a collection of sequences.
//
//  Such an input file looks like:
//
//     crab.seq:
//     ---------
//       AnalysisNumber: 1
//
//       SequencesOn: 35222
//       SequencesOff: 36817
//
//       Sequence00035222.File: sequences/sequence035222.txt
//       Sequence00036817.File: sequences/sequence036817.txt
//
//       Sequence00035222.Dir: /data2/wuerzburg/Crab-Analyse/images/035222
//       Sequence00036817.Dir: /data2/wuerzburg/Crab-Analyse/images/036817
//
// The analysis number is an artifical number used to name the output
// files automatically if the names are not overwritten in the corresponding
// programs.
//
// The sequence number are used to concatenate the filenames of the
// sequences using the file structure used in the datacenter.
//
// If you have different file names you can overwrite the default file names
// using Sequence%08d.File (make sure you have 8 digits!)
//
// In standard coditions (datacenter file system) paths are concatenated
// by using the information in the sequence files (date, etc). You can
// overwrite the directories in which the sequence-files (eg I-files) are
// stored using Sequence%08d.Dir (make sure you have 8 digits!)
//
// Resource file entries are case sensitive!
//
// MISSING (27/01/04): The default name and paths cannot be used yet, because
//                     they have to be defined soon.
//
/////////////////////////////////////////////////////////////////////////////
#include "MDataSet.h"

#include <stdlib.h>
#include <fstream>

#include <TEnv.h>
#include <TRegexp.h>
#include <TSystem.h> // TSystem::ExpandPath

#include "MLog.h"
#include "MLogManip.h"

#include "MRead.h"
#include "MAstro.h"
#include "MDirIter.h"
#include "MSequence.h"
#include "MPointingPos.h"

ClassImp(MDataSet);

using namespace std;

// --------------------------------------------------------------------------
//
// Copy the run numbers from the TString runs into the TArrayI data
//
void MDataSet::Split(TString &runs, TArrayI &data) const
{
    const TRegexp regexp("[0-9]+");

    data.Set(0);
    runs = runs.Strip(TString::kTrailing);

    while (!runs.IsNull())
    {
        TString num = runs(regexp);

        const Int_t n = data.GetSize();
        data.Set(n+1);
        data[n] = atoi(num.Data());

        runs.Remove(0, runs.First(num)+num.Length());
    }
}

void MDataSet::ResolveSequences(TEnv &env, const TArrayI &num, TList &list) const
{
    for (int i=0; i<num.GetSize(); i++)
    {
        TString name = env.GetValue(Form("Sequence%08d.File", num[i]), "");
        TString dir  = env.GetValue(Form("Sequence%08d.Dir",  num[i]), "");

        gSystem->ExpandPathName(name);
        gSystem->ExpandPathName(dir);

        // Set default sequence file and dir name
        if (name.IsNull())
            name = Form("/magic/sequences/%04d/sequence%08d.txt", num[i]/1000, num[i]);
        if (dir.IsNull())
            name = Form("/magic/data/star/%04d/%08d", num[i]/1000, num[i]);

        if (gSystem->AccessPathName(name, kFileExists))
            gLog << warn << "WARNING - Sequence file '" << name << "' doesn't exist." << endl;

        if (gSystem->AccessPathName(dir, kFileExists))
            gLog << warn << "WARNING - Directory '" << dir << "' doesn't exist." << endl;

        list.Add(new TNamed(name, dir));
    }

    // For the synchronization we must make sure, that all sequences are
    // in the correct order...
    list.Sort();
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
MDataSet::MDataSet(const char *fname)
{
    fName  = fname;

    const char *expname = gSystem->ExpandPathName(fname);

    fTitle = Form("Sequences contained in file %s", expname);

    TEnv env(expname);
    delete [] expname;

    TString str;

    fNumAnalysis = env.GetValue("AnalysisNumber",  -1);

    str = env.GetValue("SequencesOn", "");
    Split(str, fNumSequencesOn);
    str = env.GetValue("SequencesOff", "");
    Split(str, fNumSequencesOff);

    ResolveSequences(env, fNumSequencesOn,  fSequencesOn);
    ResolveSequences(env, fNumSequencesOff, fSequencesOff);


    fNameSource   = env.GetValue("SourceName", "");
    fCatalog      = env.GetValue("Catalog",    "~/Software/data/magic_favorites.edb");
    fIsWobbleMode = env.GetValue("WobbleMode", kFALSE);

    //Print();
    /*
     GetFileNames(env, fSequencesOn);
     GetFileNames(env, fSequencesOff);
     */
}

// --------------------------------------------------------------------------
//
// Return '+' if both can be accessed, '-' otherwise.
//
void MDataSet::PrintFile(const TObject &obj)
{
    const Bool_t access = !gSystem->AccessPathName(obj.GetName(), kFileExists) && !gSystem->AccessPathName(obj.GetTitle(), kFileExists) ? '+' : '-';
    gLog << " " << (access?"+":"-") << " " << obj.GetName() << " <" << obj.GetTitle() << ">" << endl;
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence
//
void MDataSet::Print(Option_t *o) const
{
    gLog << all;
    if (!IsValid())
    {
        gLog << "Sequence: " << fName << " <invalid>" << endl;
        return;
    }
    gLog << "Analysis Number: " << fNumAnalysis << endl;
    gLog << "Sequences On:   ";
    for (int i=0; i<fNumSequencesOn.GetSize(); i++)
        gLog << " " << fNumSequencesOn[i];
    gLog << endl;
    gLog << "Sequences Off:  ";
    for (int i=0; i<fNumSequencesOff.GetSize(); i++)
        gLog << " " << fNumSequencesOff[i];
    gLog << endl;

    gLog << "SourceName: " << fNameSource << endl;
    gLog << "Catalog:    " << fCatalog << endl;

    gLog << "WobbleMode: " << (fIsWobbleMode?"On":"Off") << endl;

    if (!TString(o).Contains("files", TString::kIgnoreCase))
        return;

    TObject *obj=0;

    gLog << endl;
    gLog << "On-Data Files:" << endl;
    TIter NextOn(&fSequencesOn);
    while ((obj=NextOn()))
        PrintFile(*obj);

    gLog << endl;
    gLog << "Off-Data Files:" << endl;
    TIter NextOff(&fSequencesOff);
    while ((obj=NextOff()))
        PrintFile(*obj);
}

Bool_t MDataSet::AddSequencesToList(const TList &list, MRead &read)
{
    MDirIter files;

    TIter Next(const_cast<TList*>(&list));
    TObject *o=0;
    while ((o=Next()))
    {
        MSequence seq(o->GetName());
        if (!seq.IsValid())
        {
            gLog << warn << "WARNING - Sequence " << o->GetName() << " invalid!" << endl;
            return kFALSE;
        }

        const TString dir(o->GetTitle());
        seq.SetupDatRuns(files, MSequence::kImages, dir.IsNull() ? 0 : dir.Data());
    }

    if (gLog.GetDebugLevel()>4)
    {
        gLog << dbg << "Files which are searched:" << endl;
        files.Print();
    }

    return read.AddFiles(files)>0;
}

Bool_t MDataSet::AddFiles(MRead &read) const
{
    const Bool_t rc1 = AddFilesOff(read);
    const Bool_t rc2 = AddFilesOn(read);
    return rc1 && rc2;
}

Bool_t MDataSet::AddFilesOn(MRead &read) const
{
    return AddSequencesToList(fSequencesOn, read);
}

Bool_t MDataSet::AddFilesOff(MRead &read) const
{
    return AddSequencesToList(fSequencesOff, read);
}

Bool_t MDataSet::GetSourcePos(MPointingPos &pos) const
{
    if (!HasSource())
        return kFALSE;

    TString catalog(fCatalog);
    gSystem->ExpandPathName(catalog);

    ifstream fin(catalog);
    if (!fin)
    {
        gLog << err << "Cannot open file " << catalog << ": ";
        gLog << strerror(errno) << endl;
        return kFALSE;
    }

    TString ra,dec,epoch;

    Int_t n = 0;
    while (1)
    {
        TString line;
        line.ReadLine(fin);
        if (!fin)
            break;

        n++;
        line = line.Strip(TString::kBoth);

        if (!line.BeginsWith(fNameSource))
            continue;

        // CrabNebula,f|L|K0,5:34:32.0,22:0:52,-1.0,2000

        for (int i=0; i<6; i++)
        {
            const Ssiz_t p = line.First(',');
            if (p<0 && i<5)
            {
                gLog << err << "Not enough arguments in line #" << n << endl;
                return kFALSE;
            }

            switch (i)
            {
            case 0:
            case 1:
            case 4:
                break;
            case 2:
                ra = line(0, p);
                break;
            case 3:
                dec = line(0, p);
                break;
            case 5:
                epoch = line;
                break;
            }
            line.Remove(0, p+1);
        }

        if (line.First(',')>=0)
        {
            gLog << err << "Too much arguments in line #" << n << endl;
            return kFALSE;
        }
        break;
    }

    if (epoch!=(TString)"2000")
    {
        gLog << err << "Epoch not 2000... not supported." << endl;
        return kFALSE;
    }

    Double_t r,d;
    if (!MAstro::Coordinate2Angle(ra, r))
    {
        gLog << err << "ERROR - Interpreting right ascension: " << ra << endl;
        return kFALSE;
    }
    if (!MAstro::Coordinate2Angle(dec, d))
    {
        gLog << err << "ERROR - Interpreting declination: " << dec << endl;
        return kFALSE;
    }

    pos.SetSkyPosition(r, d);
    pos.SetTitle(fNameSource);

    return kTRUE;
}