source: trunk/MagicSoft/Mars/mjobs/MSequence.cc@ 8785

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 8/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2004-2007
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MSequence
//
//  This class describes a sequence. For sequences see:
//    http://db.astro.uni-wuerzburg.de
//
//  A sequence is a collection of runs which should be used together.
//  Any run can be contained only once.
//
//  Here is an example how a file describing a sequence could look like.
//
//  Reading the file is based on TEnv. For more details see also
//  the class reference of TEnv.
//
// ===========================================================================
//
//  sequence.txt
//  ------------
//
//   # Sequence number (identifier) - necessary if the path
//   # contains the sequence number, e.g. image files
//   Sequence:     31015
//
//   # Observation Period (not needed)
//   Period:       18
//
//   # Date of sunrise of the observation night - necessary if the path
//   # contains the date, e.g. raw data
//   Night:        2004-06-24
//
//   # Start time of the sequence (first data run, not needed)
//   Start:        2004-06-24 03:12:42
//
//   # Run number of last data run in sequence (not needed)
//   LastRun:      31032
//
//   # Project name of data-runs of sequence (not needed)
//   Project:      3EG2033+41
//
//   # Source name of all runs of sequence (not needed)
//   Source:       3EG2033+41
//
//   # Trigger table of data-runs of sequence (not needed)
//   TriggerTable: L1_4NN:L2_DEFAULT
//
//   # HV Setting table of data-runs of sequence (not needed)
//   HvSettings:   HVSettings_FF36q
//
//   # Total number of data-events in sequence (not needed)
//   NumEvents:    250914
//
//   # Whether this is MC data or not (necessary in case of MCs if
//   # default paths should be used)
//   MonteCarlo: Yes
//
//   # List of all runs of this sequence (not needed)
//   Runs: 31015 31016 31017 31018 31019 31020 31021 31022 31023 31024 31025 31026 31027 31028 31029 31030 31031 31032
//
//   # List of all calibration runs of this sequence (necessary if accessed)
//   CalRuns: 31015 31016 31017
//   # List of pedestal runs belonging to the calibration runs of this sequence (necessary if accessed)
//   PedRuns: 31018
//   # List of all data runs belonging to this sequence (necessary)
//   DatRuns: 31019 31020 31022 31023 31024 31025 31027 31028 31030 31032
//
//   # Just for fun ;-) (not needed, but helpful)
//   Comment: This is a template for a sequence file
//
// ===========================================================================
//
//  For special cases you can also setup a sequence directly from a macro,
//  for example:
//
//    MDirIter pediter, datiter, caliter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddPedRuns(31751);
//    seq.AddCalRuns(31752);
//    seq.AddDatRuns(31753, 31764);
//    seq.SetupPedRuns(pediter);
//    seq.SetupCalRuns(caliter);
//    seq.SetupDatRuns(datiter);
//
//  or
//
//    MDirIter iter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddRuns(31753, 31764);
//    seq.SetupRuns(iter);
//    seq.SetupPedRuns(iter, "/mypath", "[DPC]");
//
// ===========================================================================
//
//  Class Version 2:
//   + fMonteCarlo
//
//  Class Version 3:
//   + fComment
//
//  Class Version 4:
//   + fExclRuns
//
/////////////////////////////////////////////////////////////////////////////
#include "MSequence.h"

#include <stdlib.h>

#include <TEnv.h>
#include <TRegexp.h>
#include <TSystem.h> // TSystem::ExpandPath

#include "MLog.h"
#include "MLogManip.h"

#include "MEnv.h"
#include "MJob.h"
#include "MAstro.h"
#include "MString.h"
#include "MDirIter.h"

ClassImp(MSequence);

using namespace std;

MSequence::~MSequence()
{
    /*
    TExMapIter iter(&fFileNames);

    Long_t key, val;

    while (iter.Next(key, val))
        delete (TString*)val;
        */
}


// --------------------------------------------------------------------------
//
// Copy the run numbers from the TString runs into the TArrayI data.
// Runs which are twice in the list are only added once. In this case
// a warning is emitted.
//
void MSequence::Split(TString &runs, TArrayI &data) const
{
    const TRegexp regexp("[0-9]+");

    data.Set(0);

    runs.ReplaceAll("\t", " ");
    runs = runs.Strip(TString::kBoth);

    while (!runs.IsNull())
    {
        const TString num = runs(regexp);

        if (num.IsNull())
        {
            *fLog << warn << "WARNING - Run is NaN (not a number): '" << runs << "'" << endl;
            break;
        }

        const Int_t run = atoi(num.Data());
        const Int_t n   = data.GetSize();

        // skip already existing entries
        int i;
        for (i=0; i<n; i++)
            if (data[i] == run)
                break;

        if (i<n)
            *fLog << warn << "WARNING - Run #" << run << " already in list... skipped." << endl;
        else
        {
            // set new entry
            data.Set(n+1);
            data[n] = run;
        }

        runs.Remove(0, runs.First(num)+num.Length());
    }

    MJob::SortArray(data);
}

UInt_t MSequence::SetupRuns(MDirIter &iter, const TArrayI &arr, FileType_t type, const char *path) const
{
    TString d(path);
    if (d.IsNull())
        d = fDataPath;

    const Bool_t def = d.IsNull();

    // For this particular case we assume that the files are added one by
    // one without wildcards.
    const Int_t n0 = iter.GetNumEntries();

    // Setup path
    if (def)
    {
        d = GetStandardPath();
        switch (type)
        {
        case kRawDat:  // rawdata
        case kRawPed:
        case kRawCal:
        case kRawAll:
        case kRootDat: // mcdata
        case kRootPed:
        case kRootCal:
        case kRootAll:
            d += "rawfiles/";
            d += fNight.GetStringFmt("%Y/%m/%d");
            break;
        case kCalibrated:
            d += Form("callisto/%04d/%08d", fSequence/10000, fSequence);
            break;
        case kImages:
            d += Form("star/%04d/%08d", fSequence/10000, fSequence);
            break;
        }
    }
    else
        gSystem->ExpandPathName(d);

    if (!d.EndsWith("/"))
        d += '/';

    Int_t excluded = 0;
    for (int i=0; i<arr.GetSize(); i++)
    {
        if (IsExcluded(arr[i]))
        {
            excluded++;
            continue;
        }

        // R. DeLosReyes and T. Bretz
        // Changes to read the DAQ numbering format. Changes takes place
        // between runs 35487 and 00035488 (2004_08_30)
        const char *fmt = arr[i]>35487 || fMonteCarlo ? "%08d_%s_*_E" : "%05d_%s_*_E";

        TString n;
        const char *id="_";
        switch (type)
        {
        case kRawDat:
        case kRootDat:
            id = "D";
            break;
        case kRawPed:
        case kRootPed:
            id = "P";
            break;
        case kRawCal:
        case kRootCal:
            id = "C";
            break;
        case kRawAll:
        case kRootAll:
            id = "[PCD]";
            break;
        case kCalibrated:
            id = "Y";
            break;
        case kImages:
            id = "I";
            break;
        }

        // Create file name
        n =  fNight.GetStringFmt("%Y%m%d_");
        n += Form(fmt, arr[i], id);

        switch (type)
        {
        case kRawDat:
        case kRawPed:
        case kRawCal:
        case kRawAll:
            n += ".raw.?g?z?";
            break;
        default:
            n += ".root";
        }

        // Check existance and accessibility of file
        MDirIter file(d, n, 0);
        TString name = file();
        gSystem->ExpandPathName(name);
        if (gSystem->AccessPathName(name, kFileExists))
        {
            *fLog << err;
            *fLog << "ERROR - File " << d << n << " not accessible!" << endl;
            return 0;
        }
        if (!file().IsNull())
        {
            *fLog << err;
            *fLog << "ERROR - Searching for file " << d << n << " gave more than one result!" << endl;
            return 0;
        }

        // Add Path/File to TIter
        iter.AddDirectory(d, n, 0);
    }

    // n0: The previous contents of the iter
    // n1: The number of files which have been added to the iter
    // n2: The number of files which should have been added from the array
    const Int_t n1 = iter.GetNumEntries()-n0;
    const Int_t n2 = arr.GetSize()-excluded;
    if (n1==0)
    {
        *fLog << err;
        *fLog << "ERROR - No input files for sequence #" << GetSequence() << endl;
        *fLog << "        read from " << GetName() << endl;
        *fLog << "        found in" << (def?" default-path ":" ") << d << endl;
        return 0;
    }

    if (n1==n2)
        return n1;

    *fLog << err;
    *fLog << "ERROR - " << n1 << " input files for sequence #" << GetSequence() << " found in" << endl;
    *fLog << "        " << (def?" default-path ":" ") << d << endl;
    *fLog << "        but " << n2 << " files were defined in sequence file" << endl;
    *fLog << "        " << GetName() << endl;
    if (fLog->GetDebugLevel()<=4)
        return 0;

    *fLog << inf << "Files which are searched for this sequence:" << endl;
    iter.Print();
    return 0;
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
//void MSequence::GetFileNames(TEnv &env, const TArrayI &arr)
//{
    /*
    for (int i=0; i<arr.GetSize(); i++)
    {
        // Get run number
        const Int_t num = arr[i];

        // Check if name already set
        if (fFileNames.GetValue(num))
            continue;

        TString *str = new TString(env.GetValue(Form("%d", num), ""));
        fFileNames.Add(num, (Long_t)str);
        }
        */
//}

// --------------------------------------------------------------------------
//
// Get a file name corresponding to the run-number num, returns 0 if n/a
//
//const char *MSequence::GetFileName(UInt_t num)
//{
//    return 0;
    /*
    TString *str = (TString*)fFileNames.GetValue(num);
    return str ? str->Data() : 0;*/
//}

MSequence::LightCondition_t MSequence::ReadLightCondition(TEnv &env, const char *prefix) const
{
    TString str = GetEnvValue2(env, prefix, "LightConditions", "n/a");
    if (!str.CompareTo("n/a", TString::kIgnoreCase))
        return kNA;
    if (!str.CompareTo("No_Moon", TString::kIgnoreCase))
        return kNoMoon;
    if (!str.CompareTo("Twilight", TString::kIgnoreCase))
        return kTwilight;
    if (!str.CompareTo("Moon", TString::kIgnoreCase))
        return kMoon;
    if (!str.CompareTo("Day", TString::kIgnoreCase))
        return kDay;

    gLog << warn;
    gLog << "WARNING - in " << fFileName << ":" << endl;
    gLog << "          LightCondition-tag is '" << str << "' but must be n/a, no_moon, twilight, moon or day." << endl;
    return kNA;
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
MSequence::MSequence(const char *fname, const char *path, UInt_t seq)
{
    fName  = fname;
    fTitle = path;

    fFileName = fname;
    fDataPath = path;

    gSystem->ExpandPathName(fName);
    gSystem->ExpandPathName(fTitle);

    const Bool_t rc1 = gSystem->AccessPathName(fName, kFileExists);
    const Bool_t rc2 = !fTitle.IsNull() && gSystem->AccessPathName(fTitle, kFileExists);

    if (rc1)
        gLog << err << "ERROR - Sequence file '" << fName << "' doesn't exist." << endl;
    if (rc2)
        gLog << err << "ERROR - Directory '" << fTitle << "' doesn't exist." << endl;

    MEnv env(fName);

    fSequence = (UInt_t)env.GetValue("Sequence", (Int_t)seq);
    if (rc1 || rc2)
        fSequence = (UInt_t)-1;

    const TString prefix = Form("Sequence%08d", fSequence);

    fLastRun   = GetEnvValue2(env, prefix, "LastRun",   -1);
    fNumEvents = GetEnvValue2(env, prefix, "NumEvents", -1);
    fPeriod    = GetEnvValue2(env, prefix, "Period",    -1);

    fLightCondition = ReadLightCondition(env, prefix);

    TString str;
    str = GetEnvValue2(env, prefix, "Start", "");
    fStart.SetSqlDateTime(str);
    str = GetEnvValue2(env, prefix, "Night", "");
    str += " 00:00:00";
    fNight.SetSqlDateTime(str);

    fProject      = GetEnvValue2(env, prefix, "Project", "");
    fSource       = GetEnvValue2(env, prefix, "Source", "");
    fTriggerTable = GetEnvValue2(env, prefix, "TriggerTable", "");
    fHvSettings   = GetEnvValue2(env, prefix, "HvSettings", "");
    fMonteCarlo   = GetEnvValue2(env, prefix, "MonteCarlo", kFALSE);
    fComment      = GetEnvValue2(env, prefix, "Comment",    "");

    str = GetEnvValue2(env, prefix, "Runs", "");
    Split(str, fRuns);
    str = GetEnvValue2(env, prefix, "CalRuns", "");
    Split(str, fCalRuns);
    str = GetEnvValue2(env, prefix, "PedRuns", "");
    Split(str, fPedRuns);
    str = GetEnvValue2(env, prefix, "DatRuns", "");
    Split(str, fDatRuns);
    str = GetEnvValue2(env, prefix, "Exclude", "");
    Split(str, fExclRuns);

  //  GetFileNames(env, fRuns);
  //  GetFileNames(env, fCalRuns);
  //  GetFileNames(env, fPedRuns);
  //  GetFileNames(env, fDatRuns);

    // Dummies:
    env.Touch("ZdMin");
    env.Touch("ZdMax");
    env.Touch("L1TriggerTable");
    env.Touch("L2TriggerTable");

    if (seq<0 && env.GetNumUntouched()>0)
    {
        gLog << warn << "WARNING - At least one resource in the sequence-file has not been touched!" << endl;
        env.PrintUntouched();
    }
}

//---------------------------------------------------------------------------
//
// Make sure that the name used for writing doesn't contain a full path
//
const char *MSequence::GetName() const
{
    const char *pos = strrchr(GetRcName(), '/');
    return pos>0 ? pos+1 : GetRcName();
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence
//
void MSequence::Print(Option_t *o) const
{
    const TString pre = TString(o).Contains("prefixed") ? Form("Sequence%08d.", fSequence) : "";

    gLog << all;
    if (!IsValid())
    {
        gLog << pre << "Sequence: " << fFileName << " <invalid>" << endl;
        return;
    }
    gLog << "# Path: " << GetRcName() << endl;
    gLog << "# Name: " << GetName() << endl;
    gLog << endl;
    if (pre.IsNull())
        gLog << "Sequence:       " << fSequence << endl;
    if (fMonteCarlo)
        gLog << pre << "MonteCarlo:     Yes" << endl;
    gLog << pre <<"Period:         " << fPeriod << endl;
    gLog << pre << "Night:          " << fNight << endl << endl;
    gLog << pre << "LightCondition: ";
    switch (fLightCondition)
    {
    case kNA:       gLog << "n/a" << endl;      break;
    case kNoMoon:   gLog << "NoMoon" << endl;   break;
    case kTwilight: gLog << "Twilight" << endl; break;
    case kMoon:     gLog << "Moon" << endl;     break;
    case kDay:      gLog << "Day" << endl;     break;
    }
    gLog << pre << "Start:          " << fStart << endl;
    gLog << pre << "LastRun:        " << fLastRun << endl;
    gLog << pre << "NumEvents:      " << fNumEvents << endl;
    gLog << pre << "Project:        " << fProject << endl;
    gLog << pre << "Source:         " << fSource << endl;
    gLog << pre << "TriggerTable:   " << fTriggerTable << endl;
    gLog << pre << "HvSettings:     " << fHvSettings << endl << endl;
    if (fRuns.GetSize()>0)
    {
        gLog << pre << "Runs:";
        for (int i=0; i<fRuns.GetSize(); i++)
            gLog << " " << fRuns[i];
        gLog << endl;
    }
    if (fCalRuns.GetSize()>0)
    {
        gLog << pre << "CalRuns:";
        for (int i=0; i<fCalRuns.GetSize(); i++)
            gLog << " " << fCalRuns[i];
        gLog << endl;
    }
    if (fPedRuns.GetSize()>0)
    {
        gLog << pre << "PedRuns:";
        for (int i=0; i<fPedRuns.GetSize(); i++)
            gLog << " " << fPedRuns[i];
        gLog << endl;
    }
    if (fDatRuns.GetSize()>0)
    {
        gLog << pre << "DatRuns:";
        for (int i=0; i<fDatRuns.GetSize(); i++)
            gLog << " " << fDatRuns[i];
        gLog << endl;
    }
    if (fExclRuns.GetSize()>0)
    {
        gLog << pre << "Exclude:";
        for (int i=0; i<fExclRuns.GetSize(); i++)
            gLog << " " << fExclRuns[i];
        gLog << endl;
    }
    if (!fDataPath.IsNull())
        gLog << endl << pre << "DataPath: " << fDataPath << endl;

    gLog << endl << pre << "Comment: " << fComment << endl;
}

// --------------------------------------------------------------------------
//
// Add all ped runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupPedRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fPedRuns, raw?kRawPed:kRootPed, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fDatRuns, raw?kRawDat:kRootDat, path);
}

// --------------------------------------------------------------------------
//
// Add all runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupAllRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fRuns, raw?kRawAll:kRootAll, path);
}

// --------------------------------------------------------------------------
//
// Add all calibration runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupCalRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fCalRuns, raw?kRawCal:kRootCal, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, FileType_t type, const char *path) const
{
    return SetupRuns(iter, fDatRuns, type, path);
}

// --------------------------------------------------------------------------
//
// If you want to add runs manually, use this function.
//
UInt_t MSequence::AddRuns(UInt_t first, UInt_t last, TArrayI *runs)
{
    if (last<first)
    {
        *fLog << warn << "MSequence::AddRuns - WARNING: Last runnumber " << last;
        *fLog << " smaller than first " << first << "... ignored." << endl;
        return 0;
    }
    if (!IsValid())
    {
        *fLog << inf << "Setting Sequence number to #" << first << endl;
        fSequence = first;
    }

    const UInt_t nall = fRuns.GetSize();
    const UInt_t nrun = runs ? runs->GetSize() : 0;
    const UInt_t add  = last-first+1;

    fRuns.Set(nall+add);
    if (runs)
        runs->Set(nrun+add);

    for (UInt_t i=0; i<add; i++)
    {
        fRuns[nall+i] = first+i;
        if (runs)
            (*runs)[nrun+i] = first+i;
    }
    return add;
}

Bool_t MSequence::IsContained(const TArrayI &arr, UInt_t run) const
{
    for (int i=0; i<arr.GetSize(); i++)
        if (run==(UInt_t)arr[i])
            return kTRUE;
    return kFALSE;
}

/*
// --------------------------------------------------------------------------
//
// Check if num is found in arr. If it is found remove it. Copy all
// following entries one back and decrease the array size by 1.
//
void MSequence::ExcludeRun(TArrayI &arr, UInt_t num)
{
    UInt_t *ptr = (UInt_t*)arr.GetArray();

    Int_t i = 0;
    for (i=0; i<arr.GetSize(); i++)
        if (ptr[i]==num)
            break;

    if (i==arr.GetSize())
        return;

    for (; i<arr.GetSize()-1; i++)
        ptr[i] = ptr[i+1];

    arr.Set(arr.GetSize()-1);
}
*/

// --------------------------------------------------------------------------
//
// Exclude this run (i.e. add it to fExclRuns)
//
void MSequence::ExcludeRun(UInt_t num)
{
    /*
     ExcludeRun(fRuns,    num);
     ExcludeRun(fCalRuns, num);
     ExcludeRun(fPedRuns, num);
     ExcludeRun(fDatRuns, num);
     */

    if (IsExcluded(num))
        return;

    // set new entry
    const Int_t n = fExclRuns.GetSize();
    fExclRuns.Set(n+1);
    fExclRuns[n] = num;

    MJob::SortArray(fExclRuns);
}

// --------------------------------------------------------------------------
//
// Exclude all runs which are found in the list, e.g. "100 102 105"
//
void MSequence::ExcludeRuns(TString runs)
{
    TArrayI data;
    Split(runs, data);

    for (int i=0; i<data.GetSize(); i++)
        ExcludeRun(data[i]);
}

const TString MSequence::GetExcludedRuns() const
{
    TString rc;
    for (int i=0; i<fExclRuns.GetSize(); i++)
    {
        rc += fExclRuns[i];
        rc += " ";
    }
    return rc(0, rc.Length()-1);
}

// --------------------------------------------------------------------------
//
// If you want to change or set the night manually.
// The Format is
//     SetNight("yyyy-mm-dd");
//
void MSequence::SetNight(const char *txt)
{
    TString night(txt);
    night += " 00:00:00";
    fNight.SetSqlDateTime(night);

    fPeriod = MAstro::GetMagicPeriod(fNight.GetMjd());
}

// --------------------------------------------------------------------------
//
// If the sequence name seq is just a digit it is inflated to a full
// path following the datacenter standard.
//
// Returns if file accessible or not.
//
Bool_t MSequence::InflatePath(TString &seq, Bool_t ismc)
{
    if (seq.IsDigit())
    {
        const Int_t numseq = seq.Atoi();
        seq = "/magic/";
        if (ismc)
            seq += "montecarlo/";
        seq += Form("sequences/%04d/sequence%08d.txt", numseq/10000, numseq);
        gLog << inf << "Inflated sequence file: " << seq << endl;
    }

    if (!gSystem->AccessPathName(seq, kFileExists))
        return kTRUE;

    gLog << err << "Sorry, sequence file '" << seq << "' doesn't exist." << endl;
    return kFALSE;
}