source: trunk/MagicSoft/Mars/mtemp/mucm/classes/MDataSetIter.cc@ 4693

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Marcos Lopez, 4/2004 <mailto:marcos@gae.ucm.es>
!
!   Copyright: MAGIC Software Development, 2000-2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MDataSetIter
//
//  This class serves to divide all the data taking during a given night
//  (i.e, all the root files of a given directory), 
//  into different data sets of files to be analyzed together. A data set is a
//  set of Data runs, along which their closer Ped and Cal runs.
//
//  Usage:
//  ------
//   - To specify the directory/ies where the data are use:
//        AddDirectory()
//
//   - To specify only those files for a given source name (all the files with
//     a name different from the ones in the source-name-list will be ignored)
//     use:
//        SelectSourceName()
//     You can select several src names.
//     By default, if any explict source name is specified, all are valid.
//
//   - To retrieve the next data set (Ped, Cal, and Data runs) use:
//        NextDataSet()
//        GetPefDataRuns()
//        GetCalRuns()
//        GetDatRuns()
//
// Two problems:
//    -Sources not desired : ZetaTauri
//    - Not P/C runs before data runs
//
//
/////////////////////////////////////////////////////////////////////////////
#include "MDataSetIter.h"

#include "MLog.h"
#include "MLogManip.h"

#include "MRunIter.h"

#include "TObjArray.h"

#include <TSystem.h>


ClassImp(MDataSetIter);

using namespace std;



// ----------------------------------------------------------------------------
//
//  Default constructor.
//
MDataSetIter::MDataSetIter() 
    :  fLastProcessedRun(0), fLog(&gLog), fDefCalRun(0)
{
    fPedRuns = new MRunIter;
    fCalRuns = new MRunIter;
    fDataRuns = new MRunIter;
}


// ----------------------------------------------------------------------------
//
//  Check wheather the source name is inside the list of selected source names.
//  If not any source name has been selected, by default, any source name is
//  valid.
//  Return:
//   kTRUE: this run has to be selected
//   kFALSE: this run has to be skipped
//
Int_t MDataSetIter::CheckSourceName(TString& src)
{

    // If any source has been especified, all sorce name are accepted
    if(fSrcList.GetLast()==-1)
        return kTRUE;


    //
    // For skipping Ped and Cal runs with CL (aftet June 2004)
    //
    if(src.Contains("CL") || src.Contains("ContL"))
    {
        *fLog << warn << "Skipping CL run [" << src << "]" << endl;
        return kFALSE;
    }

    //
    // For dealing with the new calibration files nomenclature 
    // (after 23-3-2004) of the type "CrabNebula-5blue", and for off data
    // like OffMrk421-1, OffMrk421-3 etc, we assume that any source name 
    // can be made up of two parts separated by a dash: the source name and 
    // a suffix indicated for instance the calibration colot of kind of off 
    // data. This suffix is then ignored for checking the source name, ie., 
    // any suffix is valid !  
    //
    if(src.Contains("-"))
    {
        *fLog << warn << "For source [" << src << "] ignoring suffix [" 
              << src( src.First("-")+1, src.Length() ) << "]" << endl;
        src.Remove(src.First("-"));
    }


    //
    // Loop 
    //
    TIter next(&fSrcList);

    TString ValidSrc;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {
        ValidSrc = n->GetName();
        
        //if (src==ValidSrc)       // For dealing with new calib files as Mrk421blue5
        if (src.Contains(ValidSrc))
            return kTRUE;
    }

    *fLog << warn << "Source [" << src << "] is not in the list of selected source names." << endl;

    return kFALSE;
}


// ---------------------------------------------------------------------------
//
//  Scan a file name. 
//
void MDataSetIter::ScanFileName(const TString& file, TString& name, TString& path, TString& date, TString& src, Int_t* run, char* type)
{
    const Int_t slash = file.Last('/');

    // Get File name and Path
    path = file(0,slash+1);
    name = file(slash+1,file.Length());
    
    // Get date
    date = name(0,8);
    
    // Get run number
    const TString runstr = name(9,5);
    *run = atoi(runstr.Data());
    
    // Get run type
    *type = name[15];

    // Get source name
    src = name(17,name.Last('_')-17);
}


// ---------------------------------------------------------------------------
//
//  Add all the files inside a given directory to the fFileList.
//
void MDataSetIter::AddToFileList(MDirIter& dir)
{
   
    TString name;
    while (!(name=dir.Next()).IsNull())
    {
        fFileList.Add((TObject*)(new TNamed((const char*)name, "")));  
    }

    //
    // Sort File List by name
    //
    fFileList.Sort();
}


// ----------------------------------------------------------------------------
//
//  Overload MDirIter::AddDirectory
//
//  Each time a new directory is added, all the files inside that directory
//  are added to the fFileList.
//
//  Returns the number of directories added
//
Int_t MDataSetIter::AddDirectory(const char *dir, const char *filter, Int_t recursive) 
{
    const Int_t rc = MDirIter::AddDirectory(dir, filter, recursive);
    if (rc==0)
        return 0;
  
    //
    // Add the files inside that directory to the fFileList. In order to 
    // add only the files inside the new directory added, we create
    // and MDirIter with only that directory. Otherwise, it we call 
    // directly this.Next() inside FillFileList the new files and the 
    // previously existing one will be again adde to the fFileList. 
    //
    MDirIter next(dir,filter,recursive);
    AddToFileList(next);
    
    return kTRUE;
} 


// ----------------------------------------------------------------------------
//
//  Add a source name to the list of selected source names.
//
void MDataSetIter::SelectSourceName(const char *src)
{
    fSrcList.Add((TObject*)(new TNamed(src, src)));  
}
    

// ----------------------------------------------------------------------------
//
// First check that previous run was not empty, and then that it contains
// runs for the current source (fSrcName)
//
Int_t MDataSetIter::IsPreviousRunUsable(MRunIter& oldRuns)
{
    if( oldRuns.GetNumRuns() == 0)
    {
        *fLog << warn << "Doesn't exist previous set." << endl;
       return kFALSE;
    }
    
    //
    // Go back to ther first entry
    //  
    oldRuns.Reset(); 
        
    //
    // Check that the previous runs were for the same source name
    //
    TString name, path, date, src;
    Int_t run;
    char type;
    
    ScanFileName(oldRuns.Next(), name, path, date, src, &run, &type);
    
   //   if( src != fSrcName)
//      {
//      *fLog << err << "Previous runs were for a diffrent source...exit." << endl;
//      return kFALSE;
//      }

        //
        // For dealing with calibration color we just ask than the srcname 
        // contains the FirstSrcName
        //
        if( !src.Contains(fSrcName) )
        {
            *fLog << warn << "Previous runs were for a diffrent source [" << src << "] vs [" << fSrcName << "] ...exit." << endl;
            return kFALSE;
        }

    
    return kTRUE;
}
    

// ---------------------------------------------------------------------------
// 
//  Assumes that a data set has the following structure:
//    P/C,P/C,P/C, ...
//    D,D,D, ...  
// 
//  After finishing the loop, it checks that were found at least one Ped, one
//  Cal and one Data runs. 
//
//  Start from the last processed run, looking for Ped, Cal and Dat runs. As
//  soon a Data run is found, it start to look only for Data runs, finishing 
//  the loop when a run of different type of Data run is found.
// 
// Process all the runs which runnumber is later to the last processed run 
//  (the first time this function is call fLastProcessedRun is 0, so it start
//  processing from the beginning)
//
//  NOTE: This FAILS if the directory doesn't start with at lead one P and one 
//        C runs  
//
Int_t MDataSetIter::NextDataSet()
{
   
    if(fFileList.GetLast()==-1)
        return kFALSE;    


    //
    // Save temporaly prvious data set;
    //
    MRunIter* oldDataRuns = 0;
    MRunIter* oldPedRuns = 0;
    MRunIter* oldCalRuns = 0;
    

    if(fDataRuns)
      oldDataRuns = (MRunIter*)fDataRuns->Clone();
    if(fPedRuns)
      oldPedRuns = (MRunIter*)fPedRuns->Clone();
    if(fCalRuns)
      oldCalRuns = (MRunIter*)fCalRuns->Clone();
    

  
    //
    // Retrieve next data set P,C and D runs
    //
    Loop("dataset");


    // 
    // If no data runs were found, is because we were already at the end of 
    // the directory. Then Return kFALSE
    //
    if ( fDataRuns->GetNumRuns()==0 ) 
    {
        *fLog << warn << "No more Data runs left." << endl;
        return kFALSE;
    }

       
    //
    // If no new P runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fPedRuns->GetNumRuns()==0 )
    {

        *fLog << warn << "No Ped runs were found before data runs." << endl;


        // 
        // Trying to Use previous pedestal, if there were
        //
        if ( IsPreviousRunUsable(*oldPedRuns) )
        {  
            *fLog << warn << "Using Previous Ped runs." << endl;
            fPedRuns->Delete();
            fPedRuns = (MRunIter*)oldPedRuns->Clone();
        }
        else
        {
            *fLog << warn << "Previous Ped runs exists but for a different source." << endl;


            //
            // Found next P after D runs
            //
            *fLog << warn << "Looking for new Ped runs after last data run..." << endl;

            oldDataRuns = (MRunIter*)fDataRuns->Clone();
            oldCalRuns = (MRunIter*)fCalRuns->Clone();
    
            Loop("P",fSrcName);
           
            // fDataRuns and fCalRuns has been overwriteen when calling Loop
            fDataRuns->Delete();
            fDataRuns = (MRunIter*)oldDataRuns->Clone();

            fCalRuns->Delete();
            fCalRuns = (MRunIter*)oldCalRuns->Clone();

            
            //
            // Last check
            //
            if ( fPedRuns->GetNumRuns() == 0 )
            {
                *fLog << err << "ERROR: Unable to found Ped runs for this source [" << fSrcName << "] ... exit." << endl;
                return kFALSE;
            }
        }
        
    }


    //
    // If no new C runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fCalRuns->GetNumRuns()==0 )
    {
        *fLog << warn << "No Cal runs were found before data runs." << endl;

        // 
        // Trying to Use previous pedestal, if there were
        //
        if ( IsPreviousRunUsable(*oldCalRuns) )
        {
            *fLog << warn << "Using Previous Cal runs." << endl;
            fCalRuns->Delete();
            fCalRuns = (MRunIter*)oldCalRuns->Clone();
        }
        else
        {

            //
            // Found next P after D runs
            //
            *fLog << warn << "Looking for new Cal runs after last data run..." << endl;
            
            
            oldDataRuns = (MRunIter*)fDataRuns->Clone();
            oldPedRuns = (MRunIter*)fPedRuns->Clone();
                
            Loop("C",fSrcName);
            
            
            // fDataRuns and fPedRuns has been overwriteen when calling Loop
            fDataRuns->Delete();
            fDataRuns = (MRunIter*)oldDataRuns->Clone();
            
            fPedRuns->Delete();
            fPedRuns = (MRunIter*)oldPedRuns->Clone();
            


            //
            // Last check
            //
            if ( fCalRuns->GetNumRuns() == 0 )
            {
                *fLog << err << "ERROR: Unable to found Cal runs for this source [" << fSrcName << "] ... exit." << endl;

                //
                // Using a default Cal run
                //
                if( fDefCalRun != 0)
                {
                    *fLog << warn << "WARN: Using a default calibration run: " << fDefCalRunPath << fDefCalRun << endl;
                    fCalRuns->AddRun(fDefCalRun,fDefCalRunPath.Data());
                }
                else
                    return kFALSE;
            }
        }
    }




    //
    // Print the runs of this data set
    //
    Print();


    oldDataRuns->Delete();
    oldPedRuns->Delete();
    oldCalRuns->Delete();
    

    return kTRUE;
}


  
// ---------------------------------------------------------------------------
//
//  Look for a set of consecutive runs of a given type (P/C/D), starting from 
//  the last processed data run. The runs found are stored in 
//  f[Data/Ped/Cal]Runs.
//
//  If option="P" look for a set of consecutive P runs. The same for C and D 
//  runs.
//  If options="dataset", look for set of consecutive D runs, but also retrieve
//  all the P and C runs between the last processed run, and the first data 
//  run.
//
//  Method:
//  -------
//  Loop over all the files, and forechs does:
//    First, performs 2 tests
//      - run number: must be larger than the last processed data run
//      - source name: must be one of the selected source names
//    then
//      - add this run, but before, check that it correspond to the same
//        src name than the previously addded runs.
//
Int_t MDataSetIter::Loop(TString option, TString LockSrcName)
{

    //
    // Delete previous data runs
    //
    fDataRuns->Delete();
    fDataRuns = new MRunIter();

    fPedRuns->Delete();
    fPedRuns = new MRunIter();

    fCalRuns->Delete();
    fCalRuns = new MRunIter();


    //
    TString FirstSrcName = "";       // SrcName of the first run found
    Bool_t RunFound     = kFALSE; 
    Bool_t DataRunFound = kFALSE; 
    Bool_t CalRunFound  = kFALSE;
    Bool_t PedRunFound  = kFALSE;


    //
    // Check option
    //
    if ( option.CompareTo("dataset",TString::kIgnoreCase) && 
         option.CompareTo("P",TString::kIgnoreCase) && 
         option.CompareTo("C",TString::kIgnoreCase) && 
         option.CompareTo("D",TString::kIgnoreCase) )
    {
        *fLog << err << "ERROR: option [" << option << "] invalid...exit" << endl;
        return kFALSE;
    }
    
    char SelectedType = !option.CompareTo("dataset",TString::kIgnoreCase) ? 'D' : option[0];
  
  


    //
    // Loop over all the files in the directory
    //
    TIter next(&fFileList);;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {  
        TString name, path, date, src;
        Int_t run;
        char type;

        ScanFileName(n->GetName(), name, path, date, src, &run, &type);

        
        // 
        // Skip until finding the following run after the fLastDataRun
        //
        if( run <= fLastProcessedRun )
            continue;
        
        //
        // Check that the source name is among one of the selected source names
        //
        if( LockSrcName != "")
        {       
            if( src != LockSrcName )
                continue;
        }
        else
        {
            if( !CheckSourceName(src) )
                continue;
        }

        //
        // If a Data run has already be found, the next files has to be only
        // of type Data, if not finish.
        //
        if( !option.CompareTo("dataset",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType )
            break;

        else if( !option.CompareTo("P",TString::kIgnoreCase) && PedRunFound==kTRUE && type!=SelectedType ) 
            break;
        
        else if( !option.CompareTo("C",TString::kIgnoreCase) && CalRunFound==kTRUE && type!=SelectedType ) 
            break;
        
        else if( !option.CompareTo("D",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType) 
            break;
        
        
        
        //
        // For Checking that this run has the same name that the first one of 
        // this set
        //
        if (RunFound==kFALSE)
        {
            RunFound = kTRUE;
            FirstSrcName = src;
        }

        if( src != FirstSrcName)
        {
            if(type!='C')
            {  
                *fLog << err << "ERROR: Source Name differs inside data set ("
                  << src << " vs. " << FirstSrcName << ") ...exit." << endl;
                return kFALSE;
            }
            else
            {
                *fLog << warn << "Warning: Cal Source Name differs inside data set (" << src << " vs. " << FirstSrcName << ")." << endl;
            }
        }


        //
        // Add this run to its corresponding MRunIter
        //
        switch (type)
        {
        case 'D':
            if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
                 !option.CompareTo("D",TString::kIgnoreCase) )
            {
                *fLog << "Adding Data run: " << run << " [" << src << "]" << endl;
        
                fDataRuns->AddRun(run,path.Data());
                
                fSrcName = src;
                fDate = date;
                
                DataRunFound = kTRUE;

                fLastProcessedRun = run;
                
            }
            break;
        
        case 'P': 
          if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
               !option.CompareTo("P",TString::kIgnoreCase)  )
            {
                *fLog << "Adding Ped run: "<< run  << " [" << src << "]" << endl;   
                
                PedRunFound = kTRUE;
                 
                fPedRuns->AddRun(run,path.Data());
            }
            break;

        case 'C':
           if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
                !option.CompareTo("C",TString::kIgnoreCase)  )
            {

              if(CalRunFound==kFALSE)
              {
                  *fLog << "Adding Cal run: "<< run  << " [" << src << "]" << endl;             
                  CalRunFound = kTRUE;
                    
                  fCalRuns->AddRun(run,path.Data());
              }
              else 
                  *fLog << "SKIPPING Cal run: "<< run  << " [" << src << "]" << endl;
              

            }
            break;
        }

        
    } // End loop


    return kTRUE;
}





// ---------------------------------------------------------------------------
//
//  By default print the P,C and D runs of the current data set. With the
//  option "all" print only all the files in the FileList.
//
void MDataSetIter::Print(const Option_t *option) const
{

    //
    //  Print all the files in the FileList.
    //
    TString s(option);
    if (s.Contains("all", TString::kIgnoreCase))
    {
        fFileList.Print();
        return;
    }

    
    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();


    //
    // Print the runs of this data set
    //
    TString file;

    *fLog << all << endl << " pedestal runs: " << endl;
    *fLog << "---------------" << endl;
    while( !(file=fPedRuns->Next()).IsNull() )
        *fLog << file << endl;

           
    *fLog << endl << " calibration runs: " << endl; 
    *fLog << "------------------" << endl;
    while( !(file=fCalRuns->Next()).IsNull() )
        *fLog << file << endl;


    *fLog << endl << " data runs: " << endl;
    *fLog << "-----------" << endl;
    while( !(file=fDataRuns->Next()).IsNull() )
        *fLog << file << endl;
    
    
    *fLog << endl;


    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();

}