source: trunk/MagicSoft/Mars/mjobs/MSequence.cc@ 9005

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 8/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2004-2008
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MSequence
//
//  This class describes a sequence. For sequences see:
//    http://db.astro.uni-wuerzburg.de
//
//  A sequence is a collection of runs which should be used together.
//  Any run can be contained only once.
//
//  Here is an example how a file describing a sequence could look like.
//
//  Reading the file is based on TEnv. For more details see also
//  the class reference of TEnv.
//
// ===========================================================================
//
//  sequence.txt
//  ------------
//
//   # Sequence number (identifier) - necessary if the path
//   # contains the sequence number, e.g. image files
//   Sequence:     31015
//
//   # Observation Period (not needed)
//   Period:       18
//
//   # Date of sunrise of the observation night - necessary if the path
//   # contains the date, e.g. raw data
//   Night:        2004-06-24
//
//   # Start time of the sequence (first data run, not needed)
//   Start:        2004-06-24 03:12:42
//
//   # Run number of last data run in sequence (not needed)
//   LastRun:      31032
//
//   # Project name of data-runs of sequence (not needed)
//   Project:      3EG2033+41
//
//   # Source name of all runs of sequence (not needed)
//   Source:       3EG2033+41
//
//   # Trigger table of data-runs of sequence (not needed)
//   TriggerTable: L1_4NN:L2_DEFAULT
//
//   # HV Setting table of data-runs of sequence (not needed)
//   HvSettings:   HVSettings_FF36q
//
//   # Total number of data-events in sequence (not needed)
//   NumEvents:    250914
//
//   # Whether this is MC data or not (necessary in case of MCs if
//   # default paths should be used)
//   MonteCarlo: Yes
//
//   # List of all runs of this sequence (not needed)
//   Runs: 31015:31017 31018 31019.0 31019.3 31019.5:7 31020+ 31021::3
//
//   # List of all calibration runs of this sequence (necessary if accessed)
//   CalRuns: 31015 31016 31017
//   # List of pedestal runs belonging to the calibration runs of this sequence (necessary if accessed)
//   PedRuns: 31018
//   # List of all data runs belonging to this sequence (necessary)
//   DatRuns: 31019.0 31019.3 31019:5:7 31020+ 31021::3
//
//   Run00031020: :3 7:9 15
//
//   # Just for fun ;-) (not needed, but helpful)
//   Comment: This is a template for a sequence file
//
// ===========================================================================
//
//   Runs are devided into file since run 1000000. These Run-/Filenumbers
//   are given by a dot. X.Y means Run X, File Y.
//
//   In the Runs, CalRuns, PedRuns and DatRuns tag you can use
//   several abbreviationa:
//
//    31015:31017     means      31015.0 31016.0 31017.0
//    31018           means      31018.0
//    31019.3         means      31019.3
//    31019.5:7       means      31019.5 31019.6 31019.7
//    31020+          means      file list for run 21020 given below
//    31021::3        means      31021.0 31021.1 31021.2 31021.3
//
//   For the run-list defined above (note the number must have 8 digits,
//   for example 'Run00031020') you can also use abbreviations:
//
//    :3              means      0 1 2 3
//    7:9             means      7 8 9
//
// ===========================================================================
//
//  For special cases you can also setup a sequence directly from a macro,
//  for example:
//
//    MDirIter pediter, datiter, caliter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddRun(31751, 'P');
//    seq.AddRun(31752, 'C');
//    seq.AddRuns(31753, 31764, 'D');
//    seq.SetupPedRuns(pediter);
//    seq.SetupCalRuns(caliter);
//    seq.SetupDatRuns(datiter);
//
//  or
//
//    MDirIter iter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddFiles(31753, 0, 120);
//    seq.SetupRuns(iter);
//    seq.SetupPedRuns(iter, "/mypath", "[DPC]");
//
//
// ===========================================================================
//
// ToDO:
//   * Default paths could be moved into the global .rootrc
//
// ===========================================================================
//
//
//  Class Version 2:
//  ----------------
//   + fMonteCarlo
//
//  Class Version 3:
//  ----------------
//   + fComment
//
//  Class Version 4:
//  ----------------
//   + fExclRuns
//
//  Class Version 5:
//  ----------------
//   + fRunsSub
//   + fDatRunsSub
//   + fPedRunsSub
//   + fCalRunsSub
//   + fExclRunsSub
//
//  Class Version 6:
//  ----------------
//   + fTelescope
//
/////////////////////////////////////////////////////////////////////////////
#include "MSequenceSQL.h"

#include <stdlib.h>
#include <errno.h>

#include <fstream>

#include <TEnv.h>
#include <TPRegexp.h>
#include <TSystem.h> // TSystem::ExpandPath

#include "MLog.h"
#include "MLogManip.h"

#include "MEnv.h"
#include "MJob.h"
#include "MAstro.h"
#include "MString.h"
#include "MDirIter.h"

ClassImp(MSequence);

using namespace std;

// --------------------------------------------------------------------------
//
// This adds an run/file entry to the data/sub arrays. If it is already
// contained a waring is printed.
//
void MSequence::AddEntry(Int_t run, Int_t file, TArrayI &data, TArrayI &sub) const
{
    const Int_t n = data.GetSize();

    // skip already existing entries
    if (IsContained(data, sub, run, file))
    {
        *fLog << warn << "WARNING - File " << run << "." << file << " already in list... skipped." << endl;
        return;
    }

    if (run<1000000 && file>0)
        *fLog << warn << "WARNING - Run number " << run << "<" << "1,000,000 but file " << file << ">0..." << endl;

    // set new entry
    data.Set(n+1);
    sub.Set(n+1);

    data[n] = run;
    sub[n]  = file;
}

// --------------------------------------------------------------------------
//
// Evaluate a token in thr Run string. Add the coresponding run/files
// with AddEntry
//
void MSequence::EvalEntry(const TEnv *env, const TString &prefix, const TString &num, TArrayI &data, TArrayI &sub) const
{
    // Split entry into run and file
    const Int_t run = num.Atoi();

    // Syntax error forbidden by construction
    const Ssiz_t p1 = num.First('.');
    const Ssiz_t p2 = num.Last(':');

    // ---------------------------------------------------------------

    const Int_t n1 = atoi(num.Data()+p1+1);
    const Int_t n2 = atoi(num.Data()+p2+1);

    // ---------------------------------------------------------------
    //  p1>=0 && p2<0  (. but no :)    run.n1     run.file1
    if (p1>=0 && p2<0)
    {
        AddEntry(run, n1, data, sub);
        return;
    }

    // ---------------------------------------------------------------
    //  p1>=0 && p2>=0  (. and :)      run:n1:n2  run.file1-run.file2
    if (p1>=0 && p2>=0)
    {
        if (n2<n1)
        {
            *fLog << warn << "WARNING - Invalid range '" << num << "'... ignored." << endl;
            return;
        }

        for (int i=n1; i<=n2; i++)
            AddEntry(run, i, data, sub);
        return;
    }

    // ---------------------------------------------------------------
    //  p1<0 && p2>=0  (no . but :)    n1:n2      run1-run2
    if (p1<0 && p2>=0)
    {
        if (n2<run)
        {
            *fLog << warn << "WARNING - Invalid range in '" << num << "'... ignored." << endl;
            return;
        }

        for (int i=run; i<=n2; i++)
            AddEntry(i, 0, data, sub);
        return;
    }

    // ---------------------------------------------------------------
    //  p0<0 and p1<0  (no . and no :)   run     run

    if (!num.EndsWith("+"))
    {
        AddEntry(run, 0, data, sub);
        return;
    }

    if (!env)
        return;

    TPRegexp regexp("([0-9]*:[0-9]+|[0-9]+(:[0-9]*)?)( +|$)");

    TString files = GetEnvValue2(*env, prefix, Form("Run%08d", run), "");
    if (files.IsNull())
    {
        AddEntry(run, 0, data, sub);
        return;
    }

    while (!files.IsNull())
    {
        const TString num = files(regexp);
        if (num.IsNull())
        {
            *fLog << warn << "WARNING - File in run " << run << " is NaN (not a number): '" << files << "'" << endl;
            break;
        }

        const Ssiz_t p1 = num.First(':');
        if (p1>=0)
        {
            const Int_t n1 = atoi(num.Data());
            const Int_t n2 = atoi(num.Data()+p1+1);

            if (n2<n1)
            {
                *fLog << warn << "WARNING - Invalid range in '" << num << "'... ignored." << endl;
                return;
            }

            // FIXME: n2==0 || n2<n1
            for (int i=n1; i<=n2; i++)
                AddEntry(run, i, data, sub);
        }
        else
        {
            const Int_t file = atoi(num.Data());
            AddEntry(run, file, data, sub);
        }

        files.Remove(0, files.First(num)+num.Length());
    }
}

// --------------------------------------------------------------------------
//
// Interprete the TString and add the run/files to the arrays.
// Run/files which are twice in the list are only added once. In this case
// a warning is emitted.
//
void MSequence::Split(TString &runs, TArrayI &data, TArrayI &sub, const TEnv *env, const TString prefix) const
{
    TPRegexp regexp("^[0-9]+([+]|(:|[.]([0-9]*:)?)[0-9]+)?( +|$)");

    data.Set(0);
    sub.Set(0);

    runs.ReplaceAll("\t", " ");
    runs = runs.Strip(TString::kBoth);

    while (!runs.IsNull())
    {
        const TString num = runs(regexp);
        if (num.IsNull())
        {
            *fLog << warn << "WARNING - Run syntax error: '" << runs << "'" << endl;
            break;
        }

        EvalEntry(env, prefix, num.Strip(TString::kTrailing), data, sub);

        runs.Remove(0, runs.First(num)+num.Length());
    }

    SortArrays(data, sub);
}

// --------------------------------------------------------------------------
//
// Get the String from the TEnv file prefixed by prefix-name.
// Intepret the string using Split which adds the run/files to the arrays.
//
void MSequence::Split(const TEnv &env, const TString &prefix, const char *name, TArrayI &data, TArrayI &sub) const
{
    TString str = GetEnvValue2(env, prefix, name, "");
    Split(str, data, sub, &env, prefix);
}

TString MSequence::InflateRunPath(const MTime &night, Bool_t mc)
{
    TString rc = GetStandardPath(mc);
    rc += "rawfiles/";
    rc += night.GetStringFmt("%Y/%m/%d/");
    return rc;
}

TString MSequence::InflateRunPath(const MTime &night, UShort_t tel, Int_t run, Int_t file, Int_t type)
{
    return InflateRunPath(night)+InflateRunName(night, tel, run, file, type);
}

// --------------------------------------------------------------------------
//
// Compile path from the given path name and expand it. If it IsNull()
// the path is compiled as standard path from tha datacenter). The
// returned path will end with a "/".
//
TString MSequence::GetPathName(TString d, FileType_t type) const
{
    // Setup path
    if (d.IsNull())
    {
        d = GetStandardPath();
        switch (type)
        {
        case kRawDat:  // rawdata
        case kRawPed:
        case kRawCal:
        case kRawAll:
        case kRootDat: // mcdata
        case kRootPed:
        case kRootCal:
        case kRootAll:
            d += "rawfiles/";
            d += fNight.GetStringFmt("%Y/%m/%d");
            break;
        case kCalibrated:
            d += Form("callisto/%04d/%08d", fSequence/10000, fSequence);
            break;
        case kImages:
            d += Form("star/%04d/%08d", fSequence/10000, fSequence);
            break;
        }
    }
    else
        gSystem->ExpandPathName(d);

    if (!d.EndsWith("/"))
        d += '/';

    return d;
}

TString MSequence::InflateRunName(const MTime &night, UShort_t tel, Int_t run, Int_t file, Int_t type)
{
    const char *id="_";
    switch (type)
    {
    case kRawDat:
    case kRootDat:
        id = "D";
        break;
    case kRawPed:
    case kRootPed:
        id = "P";
        break;
    case kRawCal:
    case kRootCal:
        id = "C";
        break;
    case kRawAll:
    case kRootAll:
        id = "[PCD]";
        break;
    case kCalibrated:
        id = "Y";
        break;
    case kImages:
        id = "I";
        break;
    }

    // ------------- Create file name --------------
    TString n = night.GetStringFmt("%Y%m%d_");

    // This is for MCs (FIXME!!!!)
    if (run<0)
        n += Form("%08d", -run);
    else
    {
        if (tel>0)
            n += Form("M%d_", tel);

        // R. DeLosReyes and T. Bretz
        // Changes to read the DAQ numbering format. Changes takes place
        // between runs 35487 and 00035488 (2004_08_30)
        n += Form(run>35487 ? "%08d" : "%05d", run);

        if (tel>0)
            n += Form(".%05d", file);
    }

    n += "_";
    n += id;
    n += "_*";

    if (tel==0)
        n += "_E";

    switch (type)
    {
    case kRawDat:
    case kRawPed:
    case kRawCal:
    case kRawAll:
        n += ".raw.?g?z?";  // TPRegexp: (\.gz)?
        break;
    default:
        n += ".root";
    }

    return n;
}

// --------------------------------------------------------------------------
//
// Return the expression describing the file-name for the file defined
// by i-th entry of the the given arrays. The file type identifier is
// defined by type. The source name is replaced by a wildcard and
// the extension is defined by the type as well.
//
TString MSequence::GetFileName(UInt_t i, const TArrayI &arr, const TArrayI &sub, FileType_t type) const
{
    return InflateRunName(fNight, arr[i]>999999?fTelescope:0, fMonteCarlo?-arr[i]:arr[i], sub.GetSize()>0?sub[i]:0, type);
}

// --------------------------------------------------------------------------
//
// Add the entries from the arrays to the MDirIter
//
UInt_t MSequence::SetupRuns(MDirIter &iter, const TArrayI &arr, const TArrayI &sub, FileType_t type, const char *path) const
{
    TString d(path);
    if (d.IsNull())
        d = fDataPath;

    const Bool_t def = d.IsNull();

    d = GetPathName(d, type);

    // For this particular case we assume that the files are added one by
    // one without wildcards.
    const Int_t n0 = iter.GetNumEntries();

    Int_t excluded = 0;
    for (int i=0; i<arr.GetSize(); i++)
    {
        if (IsExcluded(arr[i], sub[i]))
        {
            excluded++;
            continue;
        }

        const TString n = GetFileName(i, arr, sub, type);

        // Check existance and accessibility of file
        MDirIter file(d, n, 0);
        TString name = file();
        gSystem->ExpandPathName(name);
        if (gSystem->AccessPathName(name, kFileExists))
        {
            *fLog << err;
            *fLog << "ERROR - File " << d << n << " not accessible!" << endl;
            return 0;
        }
        if (!file().IsNull())
        {
            *fLog << err;
            *fLog << "ERROR - Searching for file " << d << n << " gave more than one result!" << endl;
            return 0;
        }

        // Add Path/File to TIter
        iter.AddDirectory(d, n, 0);
    }

    // n0: The previous contents of the iter
    // n1: The number of files which have been added to the iter
    // n2: The number of files which should have been added from the array
    const Int_t n1 = iter.GetNumEntries()-n0;
    const Int_t n2 = arr.GetSize()-excluded;
    if (n1==0)
    {
        *fLog << err;
        *fLog << "ERROR - No input files for sequence #" << GetSequence() << endl;
        *fLog << "        read from " << GetBaseName() << endl;
        *fLog << "        found in" << (def?" default-path ":" ") << d << endl;
        return 0;
    }

    if (n1==n2)
        return n1;

    *fLog << err;
    *fLog << "ERROR - " << n1 << " input files for sequence #" << GetSequence() << " found in" << endl;
    *fLog << "        " << (def?" default-path ":" ") << d << endl;
    *fLog << "        but " << n2 << " files were defined in sequence file" << endl;
    *fLog << "        " << GetBaseName() << endl;
    if (fLog->GetDebugLevel()<=4)
        return 0;

    *fLog << inf << "Files which are searched for this sequence:" << endl;
    iter.Print();
    return 0;
}

// --------------------------------------------------------------------------
//
// Get LightCondition from str and convert it to LightCondition_t
//
MSequence::LightCondition_t MSequence::GetLightCondition(const TString &str) const
{
    if (!str.CompareTo("n/a", TString::kIgnoreCase))
        return kNA;
    if (!str.CompareTo("No_Moon", TString::kIgnoreCase))
        return kNoMoon;
    if (!str.CompareTo("NoMoon", TString::kIgnoreCase))
        return kNoMoon;
    if (!str.CompareTo("Twilight", TString::kIgnoreCase))
        return kTwilight;
    if (!str.CompareTo("Moon", TString::kIgnoreCase))
        return kMoon;
    if (!str.CompareTo("Day", TString::kIgnoreCase))
        return kDay;

    gLog << warn;
    gLog << "WARNING - in " << GetBaseName() << ":" << endl;
    gLog << "          LightCondition-tag is '" << str << "' but must be n/a, no_moon, twilight, moon or day." << endl;
    return kNA;
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
MSequence::MSequence(const char *fname, const char *path, UInt_t seq)
{
    fName  = "MSequence";
    fTitle = "Sequence file";

    fFileName = fname;
    fDataPath = path;

    gSystem->ExpandPathName(fFileName);
    gSystem->ExpandPathName(fDataPath);

    const Bool_t rc1 = gSystem->AccessPathName(fFileName, kFileExists);
    const Bool_t rc2 = !fDataPath.IsNull() && gSystem->AccessPathName(fDataPath, kFileExists);

    if (rc1)
        gLog << err << "ERROR - Sequence file '" << fname << "' doesn't exist." << endl;
    if (rc2)
        gLog << err << "ERROR - Directory '" << path << "' doesn't exist." << endl;

    MEnv env(fFileName);

    fSequence = (UInt_t)env.GetValue("Sequence", (Int_t)seq);
    if (rc1 || rc2)
        fSequence = (UInt_t)-1;


    const TString prefix = Form("Sequence%08d", fSequence);

    fTelescope = GetEnvValue2(env, prefix, "Telescope",  1);
    fLastRun   = GetEnvValue2(env, prefix, "LastRun",   -1);
    fNumEvents = GetEnvValue2(env, prefix, "NumEvents", -1);
    fPeriod    = GetEnvValue2(env, prefix, "Period",    -1);

    TString str;
    str = GetEnvValue2(env, prefix, "LightConditions", "n/a");
    fLightCondition = GetLightCondition(str);

    str = GetEnvValue2(env, prefix, "Start", "");
    fStart.SetSqlDateTime(str);

    str = GetEnvValue2(env, prefix, "Night", "");
    str += " 00:00:00";
    fNight.SetSqlDateTime(str);

    fProject      = GetEnvValue2(env, prefix, "Project", "");
    fSource       = GetEnvValue2(env, prefix, "Source", "");
    fTriggerTable = GetEnvValue2(env, prefix, "TriggerTable", "");
    fHvSettings   = GetEnvValue2(env, prefix, "HvSettings", "");
    fMonteCarlo   = GetEnvValue2(env, prefix, "MonteCarlo", kFALSE);
    fComment      = GetEnvValue2(env, prefix, "Comment",    "");

    Split(env, prefix, "Runs",    fRuns,     fRunsSub);
    Split(env, prefix, "CalRuns", fCalRuns,  fCalRunsSub);
    Split(env, prefix, "PedRuns", fPedRuns,  fPedRunsSub);
    Split(env, prefix, "DatRuns", fDatRuns,  fDatRunsSub);
    Split(env, prefix, "Exclude", fExclRuns, fExclRunsSub);

    // Dummies:
    env.Touch("ZdMin");
    env.Touch("ZdMax");
    env.Touch("L1TriggerTable");
    env.Touch("L2TriggerTable");

    if (seq<0 && env.GetNumUntouched()>0)
    {
        gLog << warn << "WARNING - At least one resource in the sequence-file has not been touched!" << endl;
        env.PrintUntouched();
    }
}

MSequence::MSequence(const char *fname, UInt_t seq, UShort_t tel)
{
    *this = MSequenceSQL(fname, seq, tel);
}

//---------------------------------------------------------------------------
//
// Make sure that the name used for writing doesn't contain a full path
//
const char *MSequence::GetBaseName() const
{
    return gSystem->BaseName(fFileName);
}

//---------------------------------------------------------------------------
//
// Make sure that the name used for writing doesn't contain a full path
//
const char *MSequence::GetFilePath() const
{
    return gSystem->DirName(fFileName);
}

// --------------------------------------------------------------------------
//
// Find a sequence of continous numbers in f starting at pos (looking
// only at n entries). The output is converted into sequences
// of X (single r) and X:Y (a sequence between x and r). The function
// returnes when position pos+n is reached
//
TString MSequence::GetNumSequence(Int_t pos, Int_t n, const TArrayI &f) const
{
    TString str;

    Int_t p=pos;
    while (p<pos+n)
    {
        str += Form(" %d", f[p]);

        if (p==pos+n-1)
            break;

        int i=0;
        while (++i<n)
            if (f[p+i]-f[p]!=i)
                break;

        if (i>1)
            str += Form(":%d", f[p+i-1]);

        p += i;
    }

    return str;
}

// --------------------------------------------------------------------------
//
// Search for a sequence of continous numbers in r with f==0 starting at p.
// A new starting p is returned. The output is converted into sequences
// of X (single r) and X:Y (a sequence between x and r). The function
// returnes with the next f!=0.
//
TString MSequence::GetNumSequence(Int_t &p, const TArrayI &r, const TArrayI &f) const
{
    TString str;

    while (p<r.GetSize() && f[p]==0)
    {
        // serach for the first entry which doesn't fit
        // or has a file number which is != 0
        int i=0;
        while (p+ ++i<r.GetSize())
            if (r[p+i]-r[p]!=i || f[p+i]!=0)
                break;

        // None found (i-1==0)
        if (i-1==0)
            return str;

        // The last run found in the sequence (e.g. 5.0) is followed
        // by an identical run number but file != 0 (e.g. 5.1)
        if (p+i<f.GetSize() && r[p+i]==r[p] && f[p+i]!=0)
            i--;

        // Now we have to check again whether found no valid entry
        if (i-1==0)
            return str;

        str += Form(" %d", r[p]);
        // p now points to the last entry which fits and
        // has a file number == 0
        p += i-1;
        // Only one      found (i-1==1)
        // More tahn one found (i-1>1)
        str += i-1==1 ? " " : ":";
        str += Form("%d", r[p]);

        // One step forward
        p++;
    }

    return str;
}

// --------------------------------------------------------------------------
//
// Print the runs in a compressed wa. Returns the RunXXXXXXX string
// simplyfing the file setup
//
TString MSequence::PrintRuns(ostream &out, const char *pre, const char *name, const TArrayI &r, const TArrayI &f) const
{
    if (r.GetSize()==0)
        return "";

    out << pre << name;
    if (f.GetSize()==0)
        const_cast<TArrayI&>(f).Set(r.GetSize());

#ifdef DEBUG
    for (int i=0; i<r.GetSize(); i++)
        out << "  " << r[i] << "." << f[i];
    out << endl;
    return "";
#endif

    TString str;

    Int_t pos = 0;
    while (pos<r.GetSize())
    {
        TString rc = GetNumSequence(pos, r, f);
        if (!rc.IsNull())
        {
            out << rc;
            continue;
        }

        Int_t n = GetSubArray(pos, r.GetSize(), (Int_t*)r.GetArray());
        // end reached
        if (n<0)
            break;

        // This can happen if it is the last entry
        if (n==1)
        {
            out << " " << r[pos];
            if (f[pos]>0)
                out << "." << f[pos];
        }
        else
        {
            // Check for sequence
            Bool_t isseq = kTRUE;
            for (int i=1; i<n; i++)
                if (f[pos+i]-f[pos]!=i)
                    isseq=kFALSE;

            if (isseq)
            {
                out << " " << r[pos] << ".";
                if (f[pos]!=0)
                    out << f[pos];
                out << ":" << f[pos+n-1];
            }
            else
            {
                out << " " << r[pos] << "+";

                str += '\n';
                str += pre;
                str += Form("Run%08d:", r[pos]);
                str += GetNumSequence(pos, n, f);
            }
        }

        pos += n;
    }

    out << endl;

    return str;
}

// --------------------------------------------------------------------------
//
// Print the numbers in the classical way (one run after the other)
//
void MSequence::PrintRunsClassic(ostream &out, const char *pre, const char *name, const TArrayI &r) const
{
    out << pre << name;
    for (int i=0; i<r.GetSize(); i++)
        out << " " << r[i];
    out << endl;
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence
//
void MSequence::Print(ostream &out, Option_t *o) const
{
    const TString opt(o);

    const TString pre = opt.Contains("prefixed") ? Form("Sequence%08d.", fSequence) : "";

    if (!IsValid())
    {
        out << pre << "Sequence: " << fFileName << " <invalid>" << endl;
        return;
    }
    if (!fFileName.IsNull())
        out << "# FileName: " << fFileName << endl;
    if (!fDataPath.IsNull())
        out << "# DataPath: " << fDataPath << endl;
    out << endl;
    if (pre.IsNull())
        out << "Sequence:       " << fSequence << endl;
    if (fTelescope!=1)
        out << "Telescope:      " << fTelescope << endl;
    if (fMonteCarlo)
        out << pre << "MonteCarlo:     Yes" << endl;
    if (fPeriod>=0)
        out << pre << "Period:         " << fPeriod << endl;
    if (fNight!=MTime())
        out << pre << "Night:          " << fNight.GetStringFmt("%Y-%m-%d") << endl;
    out << endl;
    out << pre << "LightCondition: ";
    switch (fLightCondition)
    {
    case kNA:       out << "n/a" << endl;      break;
    case kNoMoon:   out << "NoMoon" << endl;   break;
    case kTwilight: out << "Twilight" << endl; break;
    case kMoon:     out << "Moon" << endl;     break;
    case kDay:      out << "Day" << endl;      break;
    }

    if (fStart!=MTime())
        out << pre << "Start:          " << fStart.GetSqlDateTime() << endl;
    if (fLastRun>=0)
        out << pre << "LastRun:        " << fLastRun << endl;
    if (fNumEvents>=0)
        out << pre << "NumEvents:      " << fNumEvents << endl;
    if (!fProject.IsNull())
        out << pre << "Project:        " << fProject << endl;
    if (!fSource.IsNull())
        out << pre << "Source:         " << fSource << endl;
    if (!fTriggerTable.IsNull())
        out << pre << "TriggerTable:   " << fTriggerTable << endl;
    if (!fHvSettings.IsNull())
        out << pre << "HvSettings:     " << fHvSettings << endl;
    out << endl;

    TString str;
    if (!HasSubRuns() && opt.Contains("classic"))
    {
        PrintRunsClassic(out, pre, "Runs:     ", fRuns);
        PrintRunsClassic(out, pre, "CalRuns:  ", fCalRuns);
        PrintRunsClassic(out, pre, "PedRuns:  ", fPedRuns);
        PrintRunsClassic(out, pre, "DataRuns: ", fDatRuns);
        PrintRunsClassic(out, pre, "Exclude:  ", fExclRuns);
    }
    else
    {
        str += PrintRuns(out, pre, "Runs:     ", fRuns,     fRunsSub);
        str += PrintRuns(out, pre, "CalRuns:  ", fCalRuns,  fCalRunsSub);
        str += PrintRuns(out, pre, "PedRuns:  ", fPedRuns,  fPedRunsSub);
        str += PrintRuns(out, pre, "DataRuns: ", fDatRuns,  fDatRunsSub);
        str += PrintRuns(out, pre, "Exclude:  ", fExclRuns, fExclRunsSub);
    }

    if (!fDataPath.IsNull())
        out << endl << pre << "DataPath: " << fDataPath << endl;

    if (!str.IsNull())
        out << str << endl;

    out << endl;

    if (!fComment.IsNull())
        out << pre << "Comment: " << fComment << endl;
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence to gLog
//
void MSequence::Print(Option_t *o) const
{
    gLog << all;
    Print(gLog, o);
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence to the file with name filename
//
Bool_t MSequence::WriteFile(const char *name, const Option_t *o) const
{
    ofstream fout(name);
    if (!fout)
    {
        gLog << err << "ERROR - Cannot open file " << name << ": ";
        gLog << strerror(errno) << endl;
        return kFALSE;
    }

    Print(fout, o);

    if (!fout)
    {
        gLog << err << "ERROR - Writing file " << name << ": ";
        gLog << strerror(errno) << endl;
        return kFALSE;
    }

    return kTRUE;
}

// --------------------------------------------------------------------------
//
// Add all ped runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupPedRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fPedRuns, fPedRunsSub, raw?kRawPed:kRootPed, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fDatRuns, fDatRunsSub, raw?kRawDat:kRootDat, path);
}

// --------------------------------------------------------------------------
//
// Add all runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupAllRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fRuns, fRunsSub, raw?kRawAll:kRootAll, path);
}

// --------------------------------------------------------------------------
//
// Add all calibration runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupCalRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fCalRuns, fCalRunsSub, raw?kRawCal:kRootCal, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
// Runs which are in fExlRuns are ignored.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, FileType_t type, const char *path) const
{
    return SetupRuns(iter, fDatRuns, fDatRunsSub, type, path);
}

// --------------------------------------------------------------------------
//
// check if the run/file is contained in the arrays.
//
Bool_t MSequence::IsContained(const TArrayI &arr, const TArrayI &sub, UInt_t run, UInt_t file) const
{
    // Backward compatibilty
    if (sub.GetSize()==0)
    {
        for (int i=0; i<arr.GetSize(); i++)
            if (run==(UInt_t)arr[i])
                return kTRUE;
    }
    else
    {
        for (int i=0; i<arr.GetSize(); i++)
            if (run==(UInt_t)arr[i] && file==(UInt_t)sub[i])
                return kTRUE;
    }
    return kFALSE;
}

// --------------------------------------------------------------------------
//
// Add a file (run/file) to the arrays defined by type (P, C, D, X)
//
void MSequence::AddFile(UInt_t run, UInt_t file, char type)
{
    TArrayI *r=0, *f=0;
    switch (type)
    {
    case 'P':
        r = &fPedRuns;
        f = &fPedRunsSub;
        break;
    case 'D':
        r = &fDatRuns;
        f = &fDatRunsSub;
        break;
    case 'C':
        r = &fCalRuns;
        f = &fCalRunsSub;
        break;
    default:
        r = &fRuns;
        f = &fRunsSub;
        break;
    }

    AddEntry(run, file, *r, *f);

    MJob::SortArray(fExclRuns);
}

// --------------------------------------------------------------------------
//
// Exclude this run (i.e. add it to fExclRuns)
//
void MSequence::ExcludeFile(UInt_t run, UInt_t file/*, Bool_t force*/)
{
//    if (force && IsExcluded(run, file))
//        return;

    AddEntry(run, file, fExclRuns, fExclRunsSub);

    MJob::SortArray(fExclRuns);
}

// --------------------------------------------------------------------------
//
// Exclude all runs which are found in the list, e.g. "100 102 105"
//
void MSequence::ExcludeRuns(TString runs)
{
    // FIXME: Decode stream!!!

    TArrayI data, sub;
    Split(runs, data, sub);
    for (int i=0; i<data.GetSize(); i++)
        ExcludeFile(data[i], sub[i]);
}

// --------------------------------------------------------------------------
//
// Return the excluded runs (to be more precise:the excluded files)
// in a string
//
const TString MSequence::GetExcludedRuns() const
{
    TString rc;
    for (int i=0; i<fExclRuns.GetSize(); i++)
    {
        rc += fExclRuns[i];
        if (fExclRunsSub.GetSize()>0)
        {
            rc += ".";
            rc += fExclRunsSub[i];
        }
        rc += " ";
    }
    return rc(0, rc.Length()-1);
}

// --------------------------------------------------------------------------
//
// If you want to change or set the night manually.
// The Format is
//     SetNight("yyyy-mm-dd");
//
void MSequence::SetNight(const char *txt)
{
    TString night(txt);
    night += " 00:00:00";
    fNight.SetSqlDateTime(night);

    fPeriod = MAstro::GetMagicPeriod(fNight.GetMjd());
}

// --------------------------------------------------------------------------
//
// If the sequence name seq is just a digit it is inflated to a full
// path following the datacenter standard.
//
// Returns if file accessible or not.
//
Bool_t MSequence::InflateSeq(TString &seq, Bool_t ismc)
{
    if (seq.IsDigit())
    {
        const Int_t numseq = seq.Atoi();
        seq = "/magic/";
        if (ismc)
            seq += "montecarlo/";
        seq += Form("sequences/%04d/sequence%08d.txt", numseq/10000, numseq);
        gLog << inf << "Inflated sequence file: " << seq << endl;
    }

    if (!gSystem->AccessPathName(seq, kFileExists))
        return kTRUE;

    gLog << err << "Sorry, sequence file '" << seq << "' doesn't exist." << endl;
    return kFALSE;
}

// --------------------------------------------------------------------------
//
// Search starting at position p in array arr and return the number
// of elemets which are identical to the starting entry (the starting entry
// is also counted)
//
Int_t MSequence::GetSubArray(Int_t p, Int_t n, Int_t *arr)
{
    Int_t *ptr0 = arr+p;

    Int_t *ptr = ptr0;
    Int_t *end = arr+n;

    while (ptr<end && *ptr==*ptr0)
        ptr++;

    return ptr-ptr0;
}

// --------------------------------------------------------------------------
//
// Sort the array arr2 starting at position p for following entries
// for which arr1 is equal. If not at least two entries are found which
// can be sorted return -1.
//
// The absolute index of the next entry in arr1 is returned.
//
Int_t MSequence::SortArraySub(Int_t p, Int_t n, Int_t *arr1, Int_t *arr2)
{
    Int_t *ptr2 = arr2+p;

    Int_t cnt = GetSubArray(p, n, arr1);
    if (cnt==0)
        return -1;

    TArrayI srt(cnt, ptr2);
    MJob::SortArray(srt);

    memcpy(ptr2, srt.GetArray(), srt.GetSize()*sizeof(Int_t));

    return p+srt.GetSize();
}

void MSequence::SortArrays(TArrayI &arr1, TArrayI &arr2)
{
    if (arr1.GetSize()!=arr2.GetSize())
        return;

    TArrayI idx(arr1.GetSize());

    TArrayI srt1(arr1);
    TArrayI srt2(arr2);

    TMath::Sort(arr1.GetSize(), srt1.GetArray(), idx.GetArray(), kFALSE);

    for (int i=0; i<arr1.GetSize(); i++)
    {
        arr1[i] = srt1[idx[i]];
        arr2[i] = srt2[idx[i]];
    }

    Int_t p = 0;
    while (p>=0)
        p = SortArraySub(p, arr1.GetSize(), arr1.GetArray(), arr2.GetArray());
}