source: trunk/MagicSoft/Mars/mtemp/mucm/classes/MDataSetIter.cc@ 6691

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Marcos Lopez, 4/2004 <mailto:marcos@gae.ucm.es>
!
!   Copyright: MAGIC Software Development, 2000-2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MDataSetIter
//
//  This class serves to divide all the data taking during a given night
//  (i.e, all the root files of a given directory), 
//  into different data sets of files to be analyzed together. A data set is a
//  set of Data runs, along which their closer Ped and Cal runs.
//
//  Usage:
//  ------
//   - To specify the directory/ies where the data are use:
//        AddDirectory()
//
//   - To specify only those files for a given source name (all the files with
//     a name different from the ones in the source-name-list will be ignored)
//     use:
//        SelectSourceName()
//     You can select several src names.
//     By default, if any explict source name is specified, all are valid.
//
//   - To retrieve the next data set (Ped, Cal, and Data runs) use:
//        NextDataSet()
//        GetPefDataRuns()
//        GetCalRuns()
//        GetDatRuns()
//
// Two problems:
//    -Sources not desired : ZetaTauri
//    - Not P/C runs before data runs
//
//
/////////////////////////////////////////////////////////////////////////////
#include "MDataSetIter.h"

#include "MLog.h"
#include "MLogManip.h"

#include "MRunIter.h"
#include "MSequence.h"

#include "TObjArray.h"

#include <TSystem.h>
#include <fstream>

ClassImp(MDataSetIter);

using namespace std;



// ----------------------------------------------------------------------------
//
//  Default constructor.
//
MDataSetIter::MDataSetIter() 
    : fInitialRun(-1), fEndRun(-1), fLastProcessedDataRun(0), fLastDataRun(0), fDefCalRun(0), fDefCalRunPath(""), fLog(&gLog)
{
    fPedRuns = new MRunIter;
    fCalRuns = new MRunIter;
    fDataRuns = new MRunIter;
    fSequence = new MSequence;
}


// ---------------------------------------------------------------------------
//
//  Add all the files inside a given directory to the fFileList.
//
void MDataSetIter::AddToFileList(MDirIter& dir)
{
    TString name;
   
    while (!(name=dir.Next()).IsNull())
        fFileList.Add((TObject*)(new TNamed((const char*)name, "")));
   

    //
    // Sort File List by name
    //
    fFileList.Sort();

    //
    // Find last Data runnumber of the filelist to be processed
    //
    FindLastDataRun();
}


// -------------------------------------------------------------------------
//
// Find last Data runnumber of the filelist to be processed
//
void MDataSetIter::FindLastDataRun()
{

    if(fFileList.GetLast()==-1)
        return;


    TIter next(&fFileList,kIterBackward);
    TNamed* n;

    while ( (n=(TNamed*)next()) )
    {  
        TString name, path, date, src;
        Int_t run;
        char type;
        
        ScanFileName(n->GetName(), name, path, date, src, &run, &type);

        if(type!='D')
            continue;

        
        if(fEndRun==-1)
        {
            if(run>fLastDataRun)
                fLastDataRun = run;
            
            break;
        }
        else
        {
            if(run<=fEndRun)
            {
                if(run>fLastDataRun)
                    fLastDataRun = run;
                
                break;
            }
            
        }
    }

}


// ----------------------------------------------------------------------------
//
//  Overload MDirIter::AddDirectory
//
//  Each time a new directory is added, all the files inside that directory
//  are added to the fFileList.
//
//  Returns the number of directories added
//
Int_t MDataSetIter::AddDirectory(const char *dir, const char *filter, Int_t recursive) 
{
    const Int_t rc = MDirIter::AddDirectory(dir, filter, recursive);
    if (rc==0)
        return 0;
  
    //
    // Add the files inside that directory to the fFileList. In order to 
    // add only the files inside the new directory added, we create
    // and MDirIter with only that directory. Otherwise, it we call 
    // directly this.Next() inside FillFileList the new files and the 
    // previously existing one will be again adde to the fFileList. 
    //
    MDirIter next(dir,filter,recursive);
    AddToFileList(next);

    next.Print() ;
    
    return kTRUE;
} 


// ----------------------------------------------------------------------------
//
//  Add a source name to the list of selected source names.
//
void MDataSetIter::SelectSourceName(const char *src)
{
    fSrcList.Add((TObject*)(new TNamed(src, src)));  
}
    

// ----------------------------------------------------------------------------
//
//  Check wheather the source name is inside the list of selected source names.
//  If not any source name has been selected, by default, any source name is
//  valid.
//  Return:
//   kTRUE: this run has to be selected
//   kFALSE: this run has to be skipped
//
Int_t MDataSetIter::CheckSourceName(TString& src)
{

    // If any source has been especified, all sorce name are accepted
    if(fSrcList.GetLast()==-1)
        return kTRUE;


    //
    // For skipping Ped and Cal runs with CL (aftet June 2004)
    //
    if(src.Contains("CL") || src.Contains("ContL"))
    {
        *fLog << warn << "Skipping CL run [" << src << "]" << endl;
        return kFALSE;
    }

    //
    // Skip Green and Blue calib
    //
    if(src.Contains("Blue",TString::kIgnoreCase) || src.Contains("Green",TString::kIgnoreCase))
    {
        *fLog << warn << "Skipping Blue/Green run [" << src << "]" << endl;
        return kFALSE;
    }



    //
    // For dealing with the new calibration files nomenclature 
    // (after 23-3-2004) of the type "CrabNebula-5blue", and for off data
    // like OffMrk421-1, OffMrk421-3 etc, we assume that any source name 
    // can be made up of two parts separated by a dash: the source name and 
    // a suffix indicated for instance the calibration colot of kind of off 
    // data. This suffix is then ignored for checking the source name, ie., 
    // any suffix is valid !  
    //
    if(src.Contains("-"))
    {
        //*fLog << warn << "For source [" << src << "] ignoring suffix [" 
        //     << src( src.First("-")+1, src.Length() ) << "]" << endl;
        src.Remove(src.First("-"));
    }


    //
    // Loop 
    //
    TIter next(&fSrcList);

    TString ValidSrc;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {
        ValidSrc = n->GetName();
        
        if (src==ValidSrc)       // For dealing with new calib files as Mrk421blue5
        //if (src.Contains(ValidSrc))
            return kTRUE;
    }

    *fLog << warn << "Source [" << src << "] is not in the list of selected source names." << endl;

    return kFALSE;
}

// -------------------------------------------------------------------
//      
// Compare two source names
//
Int_t MDataSetIter::CompareSourceNames(TString& src1, TString& src2)
{
    if(src1.Contains("-"))
    {
        //*fLog << warn << "For source [" << src << "] ignoring suffix ["
        //     << src( src.First("-")+1, src.Length() ) << "]" << endl;
        src1.Remove(src1.First("-"));
    }


    if(src2.Contains("-"))
    {
        //*fLog << warn << "For source [" << src << "] ignoring suffix ["
        //     << src( src.First("-")+1, src.Length() ) << "]" << endl;
        src2.Remove(src2.First("-"));
    }

    return (src1.CompareTo(src2)==0) ? kTRUE : kFALSE;
}

// ---------------------------------------------------------------------------
//
//  Scan a file name. 
//
void MDataSetIter::ScanFileName(const TString& file, TString& name, TString& path, TString& date, TString& src, Int_t* run, char* type)
{
    const Int_t slash = file.Last('/');

    // Get File name and Path
    path = file(0,slash+1);
    name = file(slash+1,file.Length());
  
    TString tmp = name;
    

    // Get date
    date = tmp(0,tmp.First('_'));
    tmp = tmp.Remove(0,tmp.First('_')+1);

 
    // Get run number
    const TString runstr = tmp(0,tmp.First('_')); 
    *run = atoi(runstr.Data());
    tmp = tmp.Remove(0,tmp.First('_')+1);


    // Get run type
    *type = tmp[0];
    tmp = tmp.Remove(0,tmp.First('_')+1);
  

    // Get source name
    //src = tmp(0,tmp.First('_')); 
    //
    // To deal with  4C15 with filemames doesn't include '_E' (18-11-2004) 
    //
    if (tmp.Contains('_'))
        src = tmp(0,tmp.First('_'));
    else        
        src = tmp(0,tmp.First('.'));
}


// ----------------------------------------------------------------------------
//
// First check that previous run was not empty, and then that it contains
// runs for the current source (fSrcName)
//
Int_t MDataSetIter::IsPreviousRunUsable(MRunIter& oldRuns)
{
    if( oldRuns.GetNumRuns() == 0)
    {
        *fLog << warn << "Doesn't exist previous set." << endl;
        return kFALSE;
    }
    
    //
    // Go back to ther first entry
    //  
    oldRuns.Reset(); 
        
    //
    // Check that the previous runs were for the same source name
    //
    TString name, path, date, src;
    Int_t run;
    char type;
    
    ScanFileName(oldRuns.Next(), name, path, date, src, &run, &type);
 
    
    if ( !CompareSourceNames(src,fSrcName) )
    //
    // For dealing with calibration color we just ask than the srcname 
    // contains the FirstSrcName
    //
    //if( !src.Contains(fSrcName) )
    {
        *fLog << warn << "Previous runs were for a diffrent source [" << src << "] vs [" << fSrcName << "] ...exit." << endl;
        return kFALSE;
    }
    
    
    return kTRUE;
}
    

// ---------------------------------------------------------------------------
// 
//  Assumes that a data set has the following structure:
//    P/C,P/C,P/C, ...
//    D,D,D, ...  
// 
//  After finishing the loop, it checks that were found at least one Ped, one
//  Cal and one Data runs. 
//
//  Start from the last processed run, looking for Ped, Cal and Dat runs. As
//  soon a Data run is found, it start to look only for Data runs, finishing 
//  the loop when a run of different type of Data run is found.
// 
// Process all the runs which runnumber is later to the last processed run 
//  (the first time this function is call fLastProcessedDataRun is 0, so it start
//  processing from the beginning)
//
//  NOTE: This FAILS if the directory doesn't start with at lead one P and one 
//        C runs  
//
Int_t MDataSetIter::NextDataSet()
{
   
    if(fFileList.GetLast()==-1)
    {
        *fLog << warn << "File list empty." << endl;
        return kFALSE;    
    }


    //
    // Save temporaly prvious data set;
    //
    MRunIter* oldDataRuns = 0;
    MRunIter* oldPedRuns = 0;
    MRunIter* oldCalRuns = 0;
    
    if(fDataRuns)
      oldDataRuns = (MRunIter*)fDataRuns->Clone();
    if(fPedRuns)
      oldPedRuns = (MRunIter*)fPedRuns->Clone();
    if(fCalRuns)
      oldCalRuns = (MRunIter*)fCalRuns->Clone();
    

  
    //
    // Retrieve next data set P,C and D runs
    //
    Loop("dataset");


    // 
    // If no data runs were found, is because we were already at the end of 
    // the directory. Then Return kFALSE
    //
    if ( fDataRuns->GetNumRuns()==0 )
    {
        *fLog << warn << "No more Data runs left." << endl;
        return kFALSE;
    }

       
    //
    // If no new P runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fPedRuns->GetNumRuns()==0 )
    {

        *fLog << warn << "No Ped runs were found before data runs." << endl;


        // 
        // Trying to Use previous pedestal, if there were
        //
        if ( IsPreviousRunUsable(*oldPedRuns) )
        {  
            *fLog << warn << "Using Previous Ped runs." << endl;
            fPedRuns->Delete();
            fPedRuns = (MRunIter*)oldPedRuns->Clone();
        }
        else
        {
            *fLog << warn << "Previous Ped runs exists but for a different source." << endl;


            //
            // Found next P after D runs
            //
            *fLog << warn << "Looking for new Ped runs after last data run..." << endl;

            oldDataRuns = (MRunIter*)fDataRuns->Clone();
            oldCalRuns = (MRunIter*)fCalRuns->Clone();
    
            Loop("P",fSrcName);
           
            // fDataRuns and fCalRuns has been overwriteen when calling Loop
            fDataRuns->Delete();
            fDataRuns = (MRunIter*)oldDataRuns->Clone();

            fCalRuns->Delete();
            fCalRuns = (MRunIter*)oldCalRuns->Clone();

            
            //
            // Last check
            //
            if ( fPedRuns->GetNumRuns() == 0 )
            {
                *fLog << err << "ERROR: Unable to found Ped runs for this source [" << fSrcName << "] ... exit." << endl;
                return kFALSE;
            }
        }
        
    }


    //
    // If no new C runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fCalRuns->GetNumRuns()==0 )
    {
        *fLog << warn << "No Cal runs were found before data runs." << endl;

        // 
        // Trying to Use previous pedestal, if there were
        //
        if ( IsPreviousRunUsable(*oldCalRuns) )
        {
            *fLog << warn << "Using Previous Cal runs." << endl;
            fCalRuns->Delete();
            fCalRuns = (MRunIter*)oldCalRuns->Clone();
        }
        else
        {

            //
            // Found next P after D runs
            //
            *fLog << warn << "Looking for new Cal runs after last data run..." << endl;
            
            
            oldDataRuns = (MRunIter*)fDataRuns->Clone();
            oldPedRuns = (MRunIter*)fPedRuns->Clone();
                
            Loop("C",fSrcName);
            
            
            // fDataRuns and fPedRuns has been overwriteen when calling Loop
            fDataRuns->Delete();
            fDataRuns = (MRunIter*)oldDataRuns->Clone();
            
            fPedRuns->Delete();
            fPedRuns = (MRunIter*)oldPedRuns->Clone();
            


            //
            // Last check
            //
            if ( fCalRuns->GetNumRuns() == 0 )
            {
                *fLog << err << "ERROR: Unable to found Cal runs for this source [" << fSrcName << "] ... exit." << endl;

                //
                // Using a default Cal run
                //
                if( fDefCalRun != 0)
                {
                    *fLog << warn << "WARN: Using a default calibration run: " << fDefCalRunPath << fDefCalRun << endl;
                    fCalRuns->AddRun(fDefCalRun,fDefCalRunPath.Data());
                }
                else
                    return kFALSE;
            }
        }
    }




    //
    // Print the runs of this data set
    //
    //Print();


    oldDataRuns->Delete();
    oldPedRuns->Delete();
    oldCalRuns->Delete();
    

    return kTRUE;
}


// -----------------------------------------------------------------------
//
//  Write an ASCII file with the sequence info. 
//  The name of the file is 'sequenceRunNum.txt' where 
//  RunNum is the run number of the first data file to analyse. We use 
//  the data runnumber instead of calib run because several sequences can
//  have the same calib run (just the P can change)
//
Int_t MDataSetIter::WriteSequence()
{
    //
    // Sequence
    //
    //fSequence->SetupDatRuns(fDataRuns, fPathData, "D");
    //fSequence->SetupPedRuns(fPedRuns, fPathData, "P");
    //fSequence->SetupCalRuns(fCalRuns, fPathData, "C");
    
    //fSequence->Print();

    TString sped =  fPedRuns->GetRunsAsFileName();
    TString scal =  fCalRuns->GetRunsAsFileName();
    TString sdata = fDataRuns->GetRunsAsFileName();

    sped.ReplaceAll("_"," ");
    scal.ReplaceAll("_"," ");
    sdata.ReplaceAll("_"," ");

   
    Int_t seqnum;
    //sscanf(scal.Data(),"%d",&seqnum);
    sscanf(sdata.Data(),"%d",&seqnum);

    TString fname = Form("sequence_%s_%d.txt",fDate.Data(), seqnum);
    ofstream out(fname);


    TString date = fDate;
    date.Insert(4,"-");
    date.Insert(7,"-");


    //out << "# Sequence number (identifier)" <<endl;
    out << "Sequence: " << seqnum << endl;
    //out << endl;
    //out << "# date of sunrise of the observation night" << endl;
    //out << "Night: 2004-09-21" << endl;
    out << "Night: " << date << endl;
    out << endl;
    //out << "# Source name of all runs of sequence" << endl;
    //out << "Source: " << fSrcName << endl;
    //out << endl;
    //out << "# List of all runs of this sequence" << endl;
    //out << "Runs: " << scal << " " << sped << " "<< sdata << endl;  
    //out << endl;
    //out << "# List of all calibration runs of this sequence" << endl;
    out << "CalRuns: " << scal << endl;
    //out << "# List of all pedestal runs of this sequence" << endl;
    out << "PedRuns: " << sped << endl;
    //out << "# List of all data runs of this sequence" << endl;
    out << "DatRuns: "<<  sdata << endl;
    out.close();

    
    return kTRUE;
}


  
// ---------------------------------------------------------------------------
//
//  Look for a set of consecutive runs of a given type (P/C/D), starting from 
//  the last processed data run. The runs found are stored in 
//  f[Data/Ped/Cal]Runs.
//
//  If option="P" look for a set of consecutive P runs. The same for C and D 
//  runs.
//  If options="dataset", look for set of consecutive D runs, but also retrieve
//  all the P and C runs between the last processed run, and the first data 
//  run.
//
//  Method:
//  -------
//  Loop over all the files, and forechs does:
//    First, performs 2 tests
//      - run number: must be larger than the last processed data run
//      - source name: must be one of the selected source names
//    then
//      - add this run, but before, check that it correspond to the same
//        src name than the previously addded runs.
//
Int_t MDataSetIter::Loop(TString option, TString LockSrcName)
{

    //
    // Delete previous data runs
    //
    fDataRuns->Delete();
    fDataRuns = new MRunIter();

    fPedRuns->Delete();
    fPedRuns = new MRunIter();

    fCalRuns->Delete();
    fCalRuns = new MRunIter();

    //
    // Init
    //
    TString FirstSrcName = "";     // SrcName of the first run found
    Bool_t RunFound     = kFALSE; 
    Bool_t DataRunFound = kFALSE; 
    Bool_t CalRunFound  = kFALSE;
    Bool_t PedRunFound  = kFALSE;

   


    //
    // Check option
    //
    if ( option.CompareTo("dataset",TString::kIgnoreCase) && 
         option.CompareTo("P",TString::kIgnoreCase) && 
         option.CompareTo("C",TString::kIgnoreCase) && 
         option.CompareTo("D",TString::kIgnoreCase) )
    {
        *fLog << err << "ERROR: option [" << option << "] invalid...exit" << endl;
        return kFALSE;
    }
    
    char SelectedType = !option.CompareTo("dataset",TString::kIgnoreCase) ? 'D' : option[0];
  
  


    //
    // Loop over all the files in the directory
    //
    TIter next(&fFileList);;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {  
        TString name, path, date, src;
        Int_t run;
        char type;

        ScanFileName(n->GetName(), name, path, date, src, &run, &type);


        //
        // Check that runnumber is inside the desire range of range to analyse [fInitialRun,fEndRun]
        //
        if(fEndRun !=-1 && run > fEndRun)
            break;
        if(run < fInitialRun)
            continue;


        // 
        // Skip until finding the following run after the last processed one, fLastDataRun
        //
        if( run <= fLastProcessedDataRun )
            continue;

        //
        // Check that the source name is among one of the selected source names
        //
        if( LockSrcName != "")
        {       
            FirstSrcName = LockSrcName;
        }
        
        if( !CheckSourceName(src) )
            continue;


        //
        // If a Data run has already be found, the next files has to be only
        // of type Data, if not finish.
        //
        if( !option.CompareTo("dataset",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType )
            break;

        else if( !option.CompareTo("P",TString::kIgnoreCase) && PedRunFound==kTRUE && type!=SelectedType ) 
            break;
        
        else if( !option.CompareTo("C",TString::kIgnoreCase) && CalRunFound==kTRUE && type!=SelectedType ) 
            break;
        
        else if( !option.CompareTo("D",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType) 
            break;
        
        
        
        //
        // For Checking that this run has the same name that the first one of 
        // this set
        //
        if (RunFound==kFALSE)
        {
            RunFound = kTRUE;
            FirstSrcName = src;
        }

        
        if( !CompareSourceNames(src,FirstSrcName) )
        {
            *fLog << err << "ERROR: Source Name differs inside data set ("
                  << src << " vs. " << FirstSrcName << ") ...exit." << endl;
            return kFALSE;
        }



        //
        // Add this run to its corresponding MRunIter
        //
        switch (type)
        {
        case 'D':
            if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
                 !option.CompareTo("D",TString::kIgnoreCase) )
            {
                *fLog << "Adding Data run: " << run << " [" << src << "]" << endl;
        
                fDataRuns->AddRun(run,path.Data());
                
                fSrcName = src;
                fDate = date;
                
                DataRunFound = kTRUE;

                fLastProcessedDataRun = run;    
            }
            break;
        
        case 'P': 
          if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
               !option.CompareTo("P",TString::kIgnoreCase)  )
            {
                *fLog << "Adding Ped run: "<< run  << " [" << src << "]" << endl;   
                
                PedRunFound = kTRUE;
                 
                fPedRuns->AddRun(run,path.Data());
            }
            break;

        case 'C':
           if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
                !option.CompareTo("C",TString::kIgnoreCase)  )
            {

              if(CalRunFound==kFALSE)
              {
                  *fLog << "Adding Cal run: "<< run  << " [" << src << "]" << endl;             
                  CalRunFound = kTRUE;
                    
                  fCalRuns->AddRun(run,path.Data());
              }
              else 
                  *fLog << "SKIPPING Cal run: "<< run  << " [" << src << "]" << endl;
              

            }
            break;
        }

        
    } // End loop


    return kTRUE;
}





// ---------------------------------------------------------------------------
//
//  By default print the P,C and D runs of the current data set. With the
//  option "all" print only all the files in the FileList.
//
void MDataSetIter::Print(const Option_t *option) const
{

    //
    //  Print all the files in the FileList.
    //
    TString s(option);
    if (s.Contains("all", TString::kIgnoreCase))
    {
        fFileList.Print();
        return;
    }

    
    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();


    //
    // Print the runs of this data set
    //
    TString file;

    *fLog << all << endl << " pedestal runs: " << endl;
    *fLog << "---------------" << endl;
    while( !(file=fPedRuns->Next()).IsNull() )
        *fLog << file << endl;

           
    *fLog << endl << " calibration runs: " << endl; 
    *fLog << "------------------" << endl;
    while( !(file=fCalRuns->Next()).IsNull() )
        *fLog << file << endl;


    *fLog << endl << " data runs: " << endl;
    *fLog << "-----------" << endl;
    while( !(file=fDataRuns->Next()).IsNull() )
        *fLog << file << endl;
    
    
    *fLog << endl;


    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();

}


// ------------------------------------------------------------------------------------
//
//  Deterimines wheather of data runs has been processed or not
//
Bool_t MDataSetIter::HasFinishedSuccesfully()
{

    if(fEndRun==-1)
        *fLog << "LastDataRun = " << fLastDataRun << endl;
    else 
        *fLog << "EndRun = " <<  fEndRun << endl;
    
    *fLog << "LastProcessedDataRun = " << fLastProcessedDataRun << endl;   


    if(fEndRun==-1)
        return fLastProcessedDataRun==fLastDataRun ? kTRUE : kFALSE;
    else 
        return fLastProcessedDataRun==fEndRun ? kTRUE : kFALSE;
       
}