source: trunk/MagicSoft/Mars/mjobs/MSequence.cc@ 7749

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 8/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MSequence
//
//  This class describes a sequence. For sequences see:
//    http://magic.astro.uni-wuerzburg.de/mars/db/queryseq.html
//
//  A sequence is a collection of runs which should be used together.
//  Any run can be contained only once.
//
//  Here is an example how a file describing a sequence could look like:
//
// ===========================================================================
//
//  sequence.txt
//  ------------
//
//   # Sequence number (identifier)
//   Sequence:     31015
//   # Observation Period (used to get the path-names)
//   Period:       18
//   # Date of sunrise of the observation night
//   Night:        2004-06-24
//
//   # Start time of the sequence (first data run)
//   Start:        2004-06-24 03:12:42
//   # Run number of last data run in sequence
//   LastRun:      31032
//   # Project name of data-runs of sequence
//   Project:      3EG2033+41
//   # Source name of all runs of sequence
//   Source:       3EG2033+41
//   # Trigger table of data-runs of sequence
//   TriggerTable: L1_4NN:L2_DEFAULT
//   # HV Setting table of data-runs of sequence
//   HvSettings:   HVSettings_FF36q
//   # Total number of data-events in sequence
//   NumEvents:    250914
//
//   # List of all runs of this sequence
//   Runs: 31015 31016 31017 31018 31019 31020 31021 31022 31023 31024 31025 31026 31027 31028 31029 31030 31031 31032
//
//   # List of all calibration runs of this sequence
//   CalRuns: 31015 31016 31017
//   # List of pedestal runs belonging to the calibration runs of this sequence
//   PedRuns: 31018
//   # List of all data runs belonging to this sequence
//   DatRuns: 31019 31020 31022 31023 31024 31025 31027 31028 31030 31032
//
//   # List of run types of all runs
//   31015: C
//   31016: C
//   31017: C
//   31018: P
//   31019: D
//   31020: D
//   31021: P
//   31022: D
//   31023: D
//   31024: D
//   31025: D
//   31026: P
//   31027: D
//   31028: D
//   31029: P
//   31030: D
//   31031: P
//   31032: D
//
// ===========================================================================
//
//  For special cases you can also setup a sequence directly from a macro,
//  for example:
//
//    MDirIter pediter, datiter, caliter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddPedRuns(31751);
//    seq.AddCalRuns(31752);
//    seq.AddDatRuns(31753, 31764);
//    seq.SetupPedRuns(pediter);
//    seq.SetupCalRuns(caliter);
//    seq.SetupDatRuns(datiter);
//
//  or
//
//    MDirIter iter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddRuns(31753, 31764);
//    seq.SetupRuns(iter);
//    seq.SetupPedRuns(iter, "/mypath", "[DPC]");
//
/////////////////////////////////////////////////////////////////////////////
#include "MSequence.h"

#include <stdlib.h>

#include <TEnv.h>
#include <TRegexp.h>
#include <TSystem.h> // TSystem::ExpandPath

#include "MLog.h"
#include "MLogManip.h"

#include "MJob.h"
#include "MAstro.h"
#include "MString.h"
#include "MDirIter.h"

ClassImp(MSequence);

using namespace std;

MSequence::~MSequence()
{
    /*
    TExMapIter iter(&fFileNames);

    Long_t key, val;

    while (iter.Next(key, val))
        delete (TString*)val;
        */
}


// --------------------------------------------------------------------------
//
// Copy the run numbers from the TString runs into the TArrayI data.
// Runs which are twice in the list are only added once. In this case
// a warning is emitted.
//
void MSequence::Split(TString &runs, TArrayI &data) const
{
    const TRegexp regexp("[0-9]+");

    data.Set(0);
    runs = runs.Strip(TString::kTrailing);

    while (!runs.IsNull())
    {
        const TString num = runs(regexp);

        if (num.IsNull())
        {
            *fLog << warn << "WARNING - Run is NaN (not a number): " << runs << endl;
            break;
        }

        const Int_t run = atoi(num.Data());
        const Int_t n   = data.GetSize();

        // skip already existing entries
        int i;
        for (i=0; i<n; i++)
            if (data[i] == run)
                break;

        if (i<n)
            *fLog << warn << "WARNING - Run #" << run << " already in list... skipped." << endl;
        else
        {
            // set new entry
            data.Set(n+1);
            data[n] = run;
        }

        runs.Remove(0, runs.First(num)+num.Length());
    }

    MJob::SortArray(data);
}

UInt_t MSequence::SetupRuns(MDirIter &iter, const TArrayI &arr, FileType_t type, const char *path) const
{
    TString d(path);

    // Setup path
    if (d.IsNull())
    {
        d = GetStandardPath();
        switch (type)
        {
        case kRawDat:
        case kRawPed:
        case kRawCal:
        case kRawAll:
            d += "rawfiles/";
            d += fNight.GetStringFmt("%Y/%m/%d");
            break;
        case kRootDat:
        case kRootPed:
        case kRootCal:
        case kRootAll:
            d += "merpp/";
            d += fNight.GetStringFmt("%Y/%m/%d");
            break;
        case kCalibrated:
            d += Form("callisto/%04d/%08d/", fSequence/10000, fSequence);
            break;
        case kImages:
            d += Form("star/%04d/%08d/", fSequence/10000, fSequence);
            break;
        }
    }
    else
        gSystem->ExpandPathName(d);

    for (int i=0; i<arr.GetSize(); i++)
    {
        // R. DeLosReyes and T. Bretz
        // Changes to read the DAQ numbering format. Changes takes place
        // between runs 35487 and 00035488 (2004_08_30)
        const char *fmt = arr[i]>35487 ? "%08d_%s_*_E" : "%05d_%s_*_E";

        TString n;
        char *id="_";
        switch (type)
        {
        case kRawDat:
        case kRootDat:
            id = "D";
            break;
        case kRawPed:
        case kRootPed:
            id = "P";
            break;
        case kRawCal:
        case kRootCal:
            id = "C";
            break;
        case kRawAll:
        case kRootAll:
            id = "[PCD]";
            break;
        case kCalibrated:
            id = "Y";
            break;
        case kImages:
            id = "I";
            break;
        }

        // Create file name
        n =  fNight.GetStringFmt("%Y%m%d_");
        n += Form(fmt, arr[i], id);

        switch (type)
        {
        case kRawDat:
        case kRawPed:
        case kRawCal:
        case kRawAll:
            n += ".raw.?g?z?";
            break;
        default:
            n += ".root";
        }

        // Add Path/File to TIter
        iter.AddDirectory(d, n, 0);
    }

    return iter.GetNumEntries();
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
void MSequence::GetFileNames(TEnv &env, const TArrayI &arr)
{
    /*
    for (int i=0; i<arr.GetSize(); i++)
    {
        // Get run number
        const Int_t num = arr[i];

        // Check if name already set
        if (fFileNames.GetValue(num))
            continue;

        TString *str = new TString(env.GetValue(Form("%d", num), ""));
        fFileNames.Add(num, (Long_t)str);
        }
        */
}

// --------------------------------------------------------------------------
//
// Get a file name corresponding to the run-number num, returns 0 if n/a
//
const char *MSequence::GetFileName(UInt_t num)
{
    return 0;
    /*
    TString *str = (TString*)fFileNames.GetValue(num);
    return str ? str->Data() : 0;*/
}

MSequence::LightCondition_t MSequence::ReadLightCondition(TEnv &env) const
{
    TString str = env.GetValue("LightCondition", "n/a");
    if (!str.CompareTo("n/a", TString::kIgnoreCase))
        return kNA;
    if (!str.CompareTo("NoMoon", TString::kIgnoreCase))
        return kNoMoon;
    if (!str.CompareTo("Twilight", TString::kIgnoreCase))
        return kTwilight;
    if (!str.CompareTo("Moon", TString::kIgnoreCase))
        return kMoon;
    if (!str.CompareTo("Day", TString::kIgnoreCase))
        return kDay;

    gLog << warn << "MSequence: LightCondition-tag not n/a, nomoon, twilight, moon or day." << endl;
    return kNA;
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
MSequence::MSequence(const char *fname)
{
    fName  = fname;

    const char *expname = gSystem->ExpandPathName(fname);

    fTitle = Form("Sequence contained in file %s", expname);

    const Bool_t access = !gSystem->AccessPathName(expname, kFileExists);
    if (!access)
        gLog << err << "ERROR - Dataset file " << expname << " not accessible!" << endl;

    TEnv env(expname);
    delete [] expname;

    TString str;

    fSequence  = env.GetValue("Sequence",  -1);
    fLastRun   = env.GetValue("LastRun",   -1);
    fNumEvents = env.GetValue("NumEvents", -1);
    fPeriod    = env.GetValue("Period",    -1);

    fLightCondition = ReadLightCondition(env);

    str = env.GetValue("Start", "");
    fStart.SetSqlDateTime(str);
    str = env.GetValue("Night", "");
    str += " 00:00:00";
    fNight.SetSqlDateTime(str);

    fProject      = env.GetValue("Project", "");
    fSource       = env.GetValue("Source", "");
    fTriggerTable = env.GetValue("TriggerTable", "");
    fHvSettings   = env.GetValue("HvSettings", "");

    str = env.GetValue("Runs", "");
    Split(str, fRuns);
    str = env.GetValue("CalRuns", "");
    Split(str, fCalRuns);
    str = env.GetValue("PedRuns", "");
    Split(str, fPedRuns);
    str = env.GetValue("DatRuns", "");
    Split(str, fDatRuns);

    GetFileNames(env, fRuns);
    GetFileNames(env, fCalRuns);
    GetFileNames(env, fPedRuns);
    GetFileNames(env, fDatRuns);
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence
//
void MSequence::Print(Option_t *o) const
{
    gLog << all;
    if (!IsValid())
    {
        gLog << "Sequence: " << fName << " <invalid>" << endl;
        return;
    }
    gLog << "Sequence:       " << fSequence << endl;
    gLog << "Period:         " << fPeriod << endl;
    gLog << "Night:          " << fNight << endl << endl;
    gLog << "LightCondition: ";
    switch (fLightCondition)
    {
    case kNA:       gLog << "n/a" << endl;      break;
    case kNoMoon:   gLog << "NoMoon" << endl;   break;
    case kTwilight: gLog << "Twilight" << endl; break;
    case kMoon:     gLog << "Moon" << endl;     break;
    case kDay:      gLog << "Day" << endl;     break;
    }
    gLog << "Start:          " << fStart << endl;
    gLog << "LastRun:        " << fLastRun << endl;
    gLog << "NumEvents:      " << fNumEvents << endl;
    gLog << "Project:        " << fProject << endl;
    gLog << "Source:         " << fSource << endl;
    gLog << "TriggerTable:   " << fTriggerTable << endl;
    gLog << "HvSettings:     " << fHvSettings << endl << endl;
    gLog << "Runs:";
    for (int i=0; i<fRuns.GetSize(); i++)
        gLog << " " << fRuns[i];
    gLog << endl;
    gLog << "CalRuns:";
    for (int i=0; i<fCalRuns.GetSize(); i++)
        gLog << " " << fCalRuns[i];
    gLog << endl;
    gLog << "PedRuns:";
    for (int i=0; i<fPedRuns.GetSize(); i++)
        gLog << " " << fPedRuns[i];
    gLog << endl;
    gLog << "DatRuns:";
    for (int i=0; i<fDatRuns.GetSize(); i++)
        gLog << " " << fDatRuns[i];
    gLog << endl;
}

// --------------------------------------------------------------------------
//
// Add all ped runs from the sequence to MDirIter.
// If path==0 the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
UInt_t MSequence::SetupPedRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fPedRuns, raw?kRawPed:kRootPed, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fDatRuns, raw?kRawDat:kRootDat, path);
}

// --------------------------------------------------------------------------
//
// Add all runs from the sequence to MDirIter.
// If path==0 the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupAllRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fRuns, raw?kRawAll:kRootAll, path);
}

// --------------------------------------------------------------------------
//
// Add all calibration runs from the sequence to MDirIter.
// If path==0 the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupCalRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fCalRuns, raw?kRawCal:kRootCal, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, FileType_t type, const char *path) const
{
    return SetupRuns(iter, fDatRuns, type, path);
}

// --------------------------------------------------------------------------
//
// If you want to add runs manually, use this function.
//
UInt_t MSequence::AddRuns(UInt_t first, UInt_t last, TArrayI *runs)
{
    if (last<first)
    {
        *fLog << warn << "MSequence::AddRuns - WARNING: Last runnumber " << last;
        *fLog << " smaller than first " << first << "... ignored." << endl;
        return 0;
    }
    if (!IsValid())
    {
        *fLog << inf << "Setting Sequence number to #" << first << endl;
        fSequence = first;
    }

    const UInt_t nall = fRuns.GetSize();
    const UInt_t nrun = runs ? runs->GetSize() : 0;
    const UInt_t add  = last-first+1;

    fRuns.Set(nall+add);
    if (runs)
        runs->Set(nrun+add);

    for (UInt_t i=0; i<add; i++)
    {
        fRuns[nall+i] = first+i;
        if (runs)
            (*runs)[nrun+i]  = first+i;
    }
    return add;
}

// --------------------------------------------------------------------------
//
// If you want to change or set the night manually.
// The Format is
//     SetNight("yyyy-mm-dd");
//
void MSequence::SetNight(const char *txt)
{
    TString night(txt);
    night += " 00:00:00";
    fNight.SetSqlDateTime(night);

    fPeriod = MAstro::GetMagicPeriod(fNight.GetMjd());
}