/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 8/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MSequence
//
//  This class describes a sequence. For sequences see:
//    http://magic.astro.uni-wuerzburg.de/mars/db/queryseq.html
//
//  A sequence is a collection of runs which should be used together.
//  Any run can be contained only once.
//
//  Here is an example how a file describing a sequence could look like:
//
// ===========================================================================
//
//  sequence.txt
//  ------------
//
//   # Sequence number (identifier)
//   Sequence:     31015
//   # Observation Period (used to get the path-names)
//   Period:       18
//   # Date of sunrise of the observation night
//   Night:        2004-06-24
//
//   # Start time of the sequence (first data run)
//   Start:        2004-06-24 03:12:42
//   # Run number of last data run in sequence
//   LastRun:      31032
//   # Project name of data-runs of sequence
//   Project:      3EG2033+41
//   # Source name of all runs of sequence
//   Source:       3EG2033+41
//   # Trigger table of data-runs of sequence
//   TriggerTable: L1_4NN:L2_DEFAULT
//   # HV Setting table of data-runs of sequence
//   HvSettings:   HVSettings_FF36q
//   # Total number of data-events in sequence
//   NumEvents:    250914
//
//   # List of all runs of this sequence
//   Runs: 31015 31016 31017 31018 31019 31020 31021 31022 31023 31024 31025 31026 31027 31028 31029 31030 31031 31032
//
//   # List of all calibration runs of this sequence
//   CalRuns: 31015 31016 31017
//   # List of pedestal runs belonging to the calibration runs of this sequence
//   PedRuns: 31018
//   # List of all data runs belonging to this sequence
//   DatRuns: 31019 31020 31022 31023 31024 31025 31027 31028 31030 31032
//
//   # List of run types of all runs
//   31015: C
//   31016: C
//   31017: C
//   31018: P
//   31019: D
//   31020: D
//   31021: P
//   31022: D
//   31023: D
//   31024: D
//   31025: D
//   31026: P
//   31027: D
//   31028: D
//   31029: P
//   31030: D
//   31031: P
//   31032: D
//
// ===========================================================================
//
//  For special cases you can also setup a sequence directly from a macro,
//  for example:
//
//    MDirIter pediter, datiter, caliter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddPedRuns(31751);
//    seq.AddCalRuns(31752);
//    seq.AddDatRuns(31753, 31764);
//    seq.SetupPedRuns(pediter);
//    seq.SetupCalRuns(caliter);
//    seq.SetupDatRuns(datiter);
//
//  or
//
//    MDirIter iter;
//
//    MSequence seq;
//    seq.SetNight("2004-07-06");
//    seq.AddRuns(31753, 31764);
//    seq.SetupRuns(iter);
//    seq.SetupPedRuns(iter, "/mypath", "[DPC]");
//
// ===========================================================================
//
//  Class Version 2:
//   + fMonteCarlo
//
/////////////////////////////////////////////////////////////////////////////
#include "MSequence.h"

#include <stdlib.h>

#include <TEnv.h>
#include <TRegexp.h>
#include <TSystem.h> // TSystem::ExpandPath

#include "MLog.h"
#include "MLogManip.h"

#include "MEnv.h"
#include "MJob.h"
#include "MAstro.h"
#include "MString.h"
#include "MDirIter.h"

ClassImp(MSequence);

using namespace std;

MSequence::~MSequence()
{
    /*
    TExMapIter iter(&fFileNames);

    Long_t key, val;

    while (iter.Next(key, val))
        delete (TString*)val;
        */
}


// --------------------------------------------------------------------------
//
// Copy the run numbers from the TString runs into the TArrayI data.
// Runs which are twice in the list are only added once. In this case
// a warning is emitted.
//
void MSequence::Split(TString &runs, TArrayI &data) const
{
    const TRegexp regexp("[0-9]+");

    data.Set(0);

    runs.ReplaceAll("\t", " ");
    runs = runs.Strip(TString::kBoth);

    while (!runs.IsNull())
    {
        const TString num = runs(regexp);

        if (num.IsNull())
        {
            *fLog << warn << "WARNING - Run is NaN (not a number): '" << runs << "'" << endl;
            break;
        }

        const Int_t run = atoi(num.Data());
        const Int_t n   = data.GetSize();

        // skip already existing entries
        int i;
        for (i=0; i<n; i++)
            if (data[i] == run)
                break;

        if (i<n)
            *fLog << warn << "WARNING - Run #" << run << " already in list... skipped." << endl;
        else
        {
            // set new entry
            data.Set(n+1);
            data[n] = run;
        }

        runs.Remove(0, runs.First(num)+num.Length());
    }

    MJob::SortArray(data);
}

UInt_t MSequence::SetupRuns(MDirIter &iter, const TArrayI &arr, FileType_t type, const char *path) const
{
    TString d(path);
    if (d.IsNull())
        d = fDataPath;

    const Bool_t def = d.IsNull();

    // For this particular case we assume that the files are added one by
    // one without wildcards.
    const Int_t n0 = iter.GetNumEntries();

    // Setup path
    if (def)
    {
        d = GetStandardPath();
        switch (type)
        {
        case kRawDat:  // rawdata
        case kRawPed:
        case kRawCal:
        case kRawAll:
        case kRootDat: // mcdata
        case kRootPed:
        case kRootCal:
        case kRootAll:
            d += "rawfiles/";
            d += fNight.GetStringFmt("%Y/%m/%d");
            break;
        case kCalibrated:
            d += Form("callisto/%04d/%08d", fSequence/10000, fSequence);
            break;
        case kImages:
            d += Form("star/%04d/%08d", fSequence/10000, fSequence);
            break;
        }
    }
    else
        gSystem->ExpandPathName(d);

    if (!d.EndsWith("/"))
        d += '/';

    for (int i=0; i<arr.GetSize(); i++)
    {
        // R. DeLosReyes and T. Bretz
        // Changes to read the DAQ numbering format. Changes takes place
        // between runs 35487 and 00035488 (2004_08_30)
        const char *fmt = arr[i]>35487 || fMonteCarlo ? "%08d_%s_*_E" : "%05d_%s_*_E";

        TString n;
        const char *id="_";
        switch (type)
        {
        case kRawDat:
        case kRootDat:
            id = "D";
            break;
        case kRawPed:
        case kRootPed:
            id = "P";
            break;
        case kRawCal:
        case kRootCal:
            id = "C";
            break;
        case kRawAll:
        case kRootAll:
            id = "[PCD]";
            break;
        case kCalibrated:
            id = "Y";
            break;
        case kImages:
            id = "I";
            break;
        }

        // Create file name
        n =  fNight.GetStringFmt("%Y%m%d_");
        n += Form(fmt, arr[i], id);

        switch (type)
        {
        case kRawDat:
        case kRawPed:
        case kRawCal:
        case kRawAll:
            n += ".raw.?g?z?";
            break;
        default:
            n += ".root";
        }

        // Check existance and accessibility of file
        MDirIter file(d, n, 0);
        TString name = file();
        gSystem->ExpandPathName(name);
        if (gSystem->AccessPathName(name, kFileExists))
        {
            *fLog << err;
            *fLog << "ERROR - File " << d << n << " not accessible!" << endl;
            return 0;
        }
        if (!file().IsNull())
        {
            *fLog << err;
            *fLog << "ERROR - Searching for file " << d << n << " gave more than one result!" << endl;
            return 0;
        }

        // Add Path/File to TIter
        iter.AddDirectory(d, n, 0);
    }

    const Int_t n1 = iter.GetNumEntries()-n0;
    const Int_t n2 = arr.GetSize();
    if (n1==0)
    {
        *fLog << err;
        *fLog << "ERROR - No input files for sequence #" << GetSequence() << endl;
        *fLog << "        read from " << GetName() << endl;
        *fLog << "        found in" << (def?" default-path ":" ") << d << endl;
        return 0;
    }

    if (n1==n2)
        return n1;

    *fLog << err;
    *fLog << "ERROR - " << n1 << " input files for sequence #" << GetSequence() << " found in" << endl;
    *fLog << "        " << (def?" default-path ":" ") << d << endl;
    *fLog << "        but " << n2 << " files were defined in sequence file" << endl;
    *fLog << "        " << GetName() << endl;
    if (fLog->GetDebugLevel()<=4)
        return 0;

    *fLog << dbg << "Files which are searched for this sequence:" << endl;
    iter.Print();
    return 0;
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
//void MSequence::GetFileNames(TEnv &env, const TArrayI &arr)
//{
    /*
    for (int i=0; i<arr.GetSize(); i++)
    {
        // Get run number
        const Int_t num = arr[i];

        // Check if name already set
        if (fFileNames.GetValue(num))
            continue;

        TString *str = new TString(env.GetValue(Form("%d", num), ""));
        fFileNames.Add(num, (Long_t)str);
        }
        */
//}

// --------------------------------------------------------------------------
//
// Get a file name corresponding to the run-number num, returns 0 if n/a
//
//const char *MSequence::GetFileName(UInt_t num)
//{
//    return 0;
    /*
    TString *str = (TString*)fFileNames.GetValue(num);
    return str ? str->Data() : 0;*/
//}

MSequence::LightCondition_t MSequence::ReadLightCondition(TEnv &env) const
{
    TString str = env.GetValue("LightConditions", "n/a");
    if (!str.CompareTo("n/a", TString::kIgnoreCase))
        return kNA;
    if (!str.CompareTo("No_Moon", TString::kIgnoreCase))
        return kNoMoon;
    if (!str.CompareTo("Twilight", TString::kIgnoreCase))
        return kTwilight;
    if (!str.CompareTo("Moon", TString::kIgnoreCase))
        return kMoon;
    if (!str.CompareTo("Day", TString::kIgnoreCase))
        return kDay;

    gLog << warn;
    gLog << "WARNING - in " << fFileName << ":" << endl;
    gLog << "          LightCondition-tag is '" << str << "' but must be n/a, no_moon, twilight, moon or day." << endl;
    return kNA;
}

// --------------------------------------------------------------------------
//
// Read the file fname as setup file for the sequence.
//
MSequence::MSequence(const char *fname, const char *path)
{
    fName  = fname;
    fTitle = path;

    fFileName = fname;
    fDataPath = path;

    gSystem->ExpandPathName(fName);
    gSystem->ExpandPathName(fTitle);

    const Bool_t rc1 = gSystem->AccessPathName(fName, kFileExists);
    const Bool_t rc2 = !fTitle.IsNull() && gSystem->AccessPathName(fTitle, kFileExists);

    if (rc1)
        gLog << err << "ERROR - Sequence file '" << fName << "' doesn't exist." << endl;
    if (rc2)
        gLog << err << "ERROR - Directory '" << fTitle << "' doesn't exist." << endl;

    MEnv env(fName);

    fSequence  = env.GetValue("Sequence", -1);
    if (rc1 || rc2)
        fSequence = (UInt_t)-1;

    fLastRun   = env.GetValue("LastRun",   -1);
    fNumEvents = env.GetValue("NumEvents", -1);
    fPeriod    = env.GetValue("Period",    -1);

    fLightCondition = ReadLightCondition(env);

    TString str;
    str = env.GetValue("Start", "");
    fStart.SetSqlDateTime(str);
    str = env.GetValue("Night", "");
    str += " 00:00:00";
    fNight.SetSqlDateTime(str);

    fProject      = env.GetValue("Project", "");
    fSource       = env.GetValue("Source", "");
    fTriggerTable = env.GetValue("TriggerTable", "");
    fHvSettings   = env.GetValue("HvSettings", "");
    fMonteCarlo   = env.GetValue("MonteCarlo", kFALSE);

    str = env.GetValue("Runs", "");
    Split(str, fRuns);
    str = env.GetValue("CalRuns", "");
    Split(str, fCalRuns);
    str = env.GetValue("PedRuns", "");
    Split(str, fPedRuns);
    str = env.GetValue("DatRuns", "");
    Split(str, fDatRuns);

  //  GetFileNames(env, fRuns);
  //  GetFileNames(env, fCalRuns);
  //  GetFileNames(env, fPedRuns);
  //  GetFileNames(env, fDatRuns);

    // Dummies:
    env.GetValue("ZdMin", 0);
    env.GetValue("ZdMax", 0);
    env.GetValue("L1TriggerTable", 0);
    env.GetValue("L2TriggerTable", 0);

    if (env.GetNumUntouched()>0)
    {
        gLog << warn << "WARNING - At least one resource in the dataset-file has not been touched!" << endl;
        env.PrintUntouched();
    }
}

// --------------------------------------------------------------------------
//
// Print the contents of the sequence
//
void MSequence::Print(Option_t *o) const
{
    gLog << all;
    if (!IsValid())
    {
        gLog << "Sequence: " << fFileName << " <invalid>" << endl;
        return;
    }
    gLog << "Sequence:       " << fSequence << endl;
    if (fMonteCarlo)
        gLog << "MonteCarlo:     Yes" << endl;
    gLog << "Period:         " << fPeriod << endl;
    gLog << "Night:          " << fNight << endl << endl;
    gLog << "LightCondition: ";
    switch (fLightCondition)
    {
    case kNA:       gLog << "n/a" << endl;      break;
    case kNoMoon:   gLog << "NoMoon" << endl;   break;
    case kTwilight: gLog << "Twilight" << endl; break;
    case kMoon:     gLog << "Moon" << endl;     break;
    case kDay:      gLog << "Day" << endl;     break;
    }
    gLog << "Start:          " << fStart << endl;
    gLog << "LastRun:        " << fLastRun << endl;
    gLog << "NumEvents:      " << fNumEvents << endl;
    gLog << "Project:        " << fProject << endl;
    gLog << "Source:         " << fSource << endl;
    gLog << "TriggerTable:   " << fTriggerTable << endl;
    gLog << "HvSettings:     " << fHvSettings << endl << endl;
    gLog << "Runs:";
    for (int i=0; i<fRuns.GetSize(); i++)
        gLog << " " << fRuns[i];
    gLog << endl;
    gLog << "CalRuns:";
    for (int i=0; i<fCalRuns.GetSize(); i++)
        gLog << " " << fCalRuns[i];
    gLog << endl;
    gLog << "PedRuns:";
    for (int i=0; i<fPedRuns.GetSize(); i++)
        gLog << " " << fPedRuns[i];
    gLog << endl;
    gLog << "DatRuns:";
    for (int i=0; i<fDatRuns.GetSize(); i++)
        gLog << " " << fDatRuns[i];
    gLog << endl;

    if (!fDataPath.IsNull())
        gLog << endl << "DataPath: " << fDataPath << endl;
}

// --------------------------------------------------------------------------
//
// Add all ped runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupPedRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fPedRuns, raw?kRawPed:kRootPed, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fDatRuns, raw?kRawDat:kRootDat, path);
}

// --------------------------------------------------------------------------
//
// Add all runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupAllRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fRuns, raw?kRawAll:kRootAll, path);
}

// --------------------------------------------------------------------------
//
// Add all calibration runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupCalRuns(MDirIter &iter, const char *path, Bool_t raw) const
{
    return SetupRuns(iter, fCalRuns, raw?kRawCal:kRootCal, path);
}

// --------------------------------------------------------------------------
//
// Add all data runs from the sequence to MDirIter.
// If path==0 fDataPath is used instead. If it is also empty
// the standard path of the data-center is assumed.
// If you have the runs locally use path="."
// Using raw=kTRUE you get correspodning raw-files setup.
// Return the number of files added.
//
UInt_t MSequence::SetupDatRuns(MDirIter &iter, FileType_t type, const char *path) const
{
    return SetupRuns(iter, fDatRuns, type, path);
}

// --------------------------------------------------------------------------
//
// If you want to add runs manually, use this function.
//
UInt_t MSequence::AddRuns(UInt_t first, UInt_t last, TArrayI *runs)
{
    if (last<first)
    {
        *fLog << warn << "MSequence::AddRuns - WARNING: Last runnumber " << last;
        *fLog << " smaller than first " << first << "... ignored." << endl;
        return 0;
    }
    if (!IsValid())
    {
        *fLog << inf << "Setting Sequence number to #" << first << endl;
        fSequence = first;
    }

    const UInt_t nall = fRuns.GetSize();
    const UInt_t nrun = runs ? runs->GetSize() : 0;
    const UInt_t add  = last-first+1;

    fRuns.Set(nall+add);
    if (runs)
        runs->Set(nrun+add);

    for (UInt_t i=0; i<add; i++)
    {
        fRuns[nall+i] = first+i;
        if (runs)
            (*runs)[nrun+i]  = first+i;
    }
    return add;
}

// --------------------------------------------------------------------------
//
// If you want to change or set the night manually.
// The Format is
//     SetNight("yyyy-mm-dd");
//
void MSequence::SetNight(const char *txt)
{
    TString night(txt);
    night += " 00:00:00";
    fNight.SetSqlDateTime(night);

    fPeriod = MAstro::GetMagicPeriod(fNight.GetMjd());
}

// --------------------------------------------------------------------------
//
// If the sequence name seq is just a digit it is inflated to a full
// path following the datacenter standard.
//
// Returns if file accessible or not.
//
Bool_t MSequence::InflatePath(TString &seq, Bool_t ismc)
{
    if (seq.IsDigit())
    {
        const Int_t numseq = seq.Atoi();
        seq = "/magic/";
        if (ismc)
            seq += "montecarlo/";
        seq += Form("sequences/%04d/sequence%08d.txt", numseq/10000, numseq);
        gLog << inf << "Inflated sequence file: " << seq << endl;
    }

    if (!gSystem->AccessPathName(seq, kFileExists))
        return kTRUE;

    gLog << err << "Sorry, sequence file '" << seq << "' doesn't exist." << endl;
    return kFALSE;
}
