/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Marcos Lopez, 4/2004 <mailto:marcos@gae.ucm.es>
!
!   Copyright: MAGIC Software Development, 2000-2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MDataSetIter
//
//  This class serves to divide all the data taking during a given night
//  (i.e, all the root files of a given directory), 
//  into different data sets of files to be analyzed together. A data set is a
//  set of Data runs, along which their closer Ped and Cal runs.
//
//  Usage:
//  ------
//   - To specify the directory/ies where the data are use:
//        AddDirectory()
//
//   - To specify only those files for a given source name (all the files with
//     a name different from the ones in the source-name-list will be ignored)
//     use:
//        SelectSourceName()
//     You can select several src names.
//     By default, if any explict source name is specified, all are valid.
//
//   - To retrieve the next data set (Ped, Cal, and Data runs) use:
//        NextDataSet()
//        GetPefDataRuns()
//        GetCalRuns()
//        GetDatRuns()
//
// Two problems:
//    -Sources not desired : ZetaTauri
//    - Not P/C runs before data runs
//
//
/////////////////////////////////////////////////////////////////////////////
#include "MDataSetIter.h"

#include "MLog.h"
#include "MLogManip.h"

#include "MRunIter.h"

#include "TObjArray.h"

#include <TSystem.h>


ClassImp(MDataSetIter);

using namespace std;



// ----------------------------------------------------------------------------
//
//  Default constructor.
//
MDataSetIter::MDataSetIter() 
    :  fLastProcessedRun(0), fLog(&gLog), fDefCalRun(0)
{
    fPedRuns = new MRunIter;
    fCalRuns = new MRunIter;
    fDataRuns = new MRunIter;
}


// ----------------------------------------------------------------------------
//
//  Check wheather the source name is inside the list of selected source names.
//  If not any source name has been selected, by default, any source name is
//  valid.
//  Return:
//   kTRUE: this run has to be selected
//   kFALSE: this run has to be skipped
//
Int_t MDataSetIter::CheckSourceName(TString& src)
{

    // If any source has been especified, all sorce name are accepted
    if(fSrcList.GetLast()==-1)
	return kTRUE;


    //
    // For skipping Ped and Cal runs with CL (aftet June 2004)
    //
    if(src.Contains("CL") || src.Contains("ContL"))
    {
        *fLog << warn << "Skipping CL run [" << src << "]" << endl;
	return kFALSE;
    }

    //
    // For dealing with the new calibration files nomenclature 
    // (after 23-3-2004) of the type "CrabNebula-5blue", and for off data
    // like OffMrk421-1, OffMrk421-3 etc, we assume that any source name 
    // can be made up of two parts separated by a dash: the source name and 
    // a suffix indicated for instance the calibration colot of kind of off 
    // data. This suffix is then ignored for checking the source name, ie., 
    // any suffix is valid !  
    //
    if(src.Contains("-"))
    {
	*fLog << warn << "For source [" << src << "] ignoring suffix [" 
	      << src( src.First("-")+1, src.Length() ) << "]" << endl;
	src.Remove(src.First("-"));
    }


    //
    // Loop 
    //
    TIter next(&fSrcList);

    TString ValidSrc;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {
	ValidSrc = n->GetName();
	
	//if (src==ValidSrc)       // For dealing with new calib files as Mrk421blue5
	if (src.Contains(ValidSrc))
	    return kTRUE;
    }

    *fLog << warn << "Source [" << src << "] is not in the list of selected source names." << endl;

    return kFALSE;
}


// ---------------------------------------------------------------------------
//
//  Scan a file name. 
//
void MDataSetIter::ScanFileName(const TString& file, TString& name, TString& path, TString& date, TString& src, Int_t* run, char* type)
{
    const Int_t slash = file.Last('/');

    // Get File name and Path
    path = file(0,slash+1);
    name = file(slash+1,file.Length());
    
    // Get date
    date = name(0,8);
    
    // Get run number
    const TString runstr = name(9,5);
    *run = atoi(runstr.Data());
    
    // Get run type
    *type = name[15];

    // Get source name
    src = name(17,name.Last('_')-17);
}


// ---------------------------------------------------------------------------
//
//  Add all the files inside a given directory to the fFileList.
//
void MDataSetIter::AddToFileList(MDirIter& dir)
{
   
    TString name;
    while (!(name=dir.Next()).IsNull())
    {
	fFileList.Add((TObject*)(new TNamed((const char*)name, "")));  
    }

    //
    // Sort File List by name
    //
    fFileList.Sort();
}


// ----------------------------------------------------------------------------
//
//  Overload MDirIter::AddDirectory
//
//  Each time a new directory is added, all the files inside that directory
//  are added to the fFileList.
//
//  Returns the number of directories added
//
Int_t MDataSetIter::AddDirectory(const char *dir, const char *filter, Int_t recursive) 
{
    const Int_t rc = MDirIter::AddDirectory(dir, filter, recursive);
    if (rc==0)
	return 0;
  
    //
    // Add the files inside that directory to the fFileList. In order to 
    // add only the files inside the new directory added, we create
    // and MDirIter with only that directory. Otherwise, it we call 
    // directly this.Next() inside FillFileList the new files and the 
    // previously existing one will be again adde to the fFileList. 
    //
    MDirIter next(dir,filter,recursive);
    AddToFileList(next);
    
    return kTRUE;
} 


// ----------------------------------------------------------------------------
//
//  Add a source name to the list of selected source names.
//
void MDataSetIter::SelectSourceName(const char *src)
{
    fSrcList.Add((TObject*)(new TNamed(src, src)));  
}
    

// ----------------------------------------------------------------------------
//
// First check that previous run was not empty, and then that it contains
// runs for the current source (fSrcName)
//
Int_t MDataSetIter::IsPreviousRunUsable(MRunIter& oldRuns)
{
    if( oldRuns.GetNumRuns() == 0)
    {
	*fLog << warn << "Doesn't exist previous set." << endl;
       return kFALSE;
    }
    
    //
    // Go back to ther first entry
    //	
    oldRuns.Reset(); 
	
    //
    // Check that the previous runs were for the same source name
    //
    TString name, path, date, src;
    Int_t run;
    char type;
    
    ScanFileName(oldRuns.Next(), name, path, date, src, &run, &type);
    
   //   if( src != fSrcName)
//      {
//  	*fLog << err << "Previous runs were for a diffrent source...exit." << endl;
//  	return kFALSE;
//      }

	//
	// For dealing with calibration color we just ask than the srcname 
	// contains the FirstSrcName
	//
	if( !src.Contains(fSrcName) )
	{
	    *fLog << warn << "Previous runs were for a diffrent source [" << src << "] vs [" << fSrcName << "] ...exit." << endl;
	    return kFALSE;
	}

    
    return kTRUE;
}
    

// ---------------------------------------------------------------------------
// 
//  Assumes that a data set has the following structure:
//    P/C,P/C,P/C, ...
//    D,D,D, ...  
// 
//  After finishing the loop, it checks that were found at least one Ped, one
//  Cal and one Data runs. 
//
//  Start from the last processed run, looking for Ped, Cal and Dat runs. As
//  soon a Data run is found, it start to look only for Data runs, finishing 
//  the loop when a run of different type of Data run is found.
// 
// Process all the runs which runnumber is later to the last processed run 
//  (the first time this function is call fLastProcessedRun is 0, so it start
//  processing from the beginning)
//
//  NOTE: This FAILS if the directory doesn't start with at lead one P and one 
//        C runs  
//
Int_t MDataSetIter::NextDataSet()
{
   
    if(fFileList.GetLast()==-1)
	return kFALSE;    


    //
    // Save temporaly prvious data set;
    //
    MRunIter* oldDataRuns = 0;
    MRunIter* oldPedRuns = 0;
    MRunIter* oldCalRuns = 0;
    

    if(fDataRuns)
      oldDataRuns = (MRunIter*)fDataRuns->Clone();
    if(fPedRuns)
      oldPedRuns = (MRunIter*)fPedRuns->Clone();
    if(fCalRuns)
      oldCalRuns = (MRunIter*)fCalRuns->Clone();
    

  
    //
    // Retrieve next data set P,C and D runs
    //
    Loop("dataset");


    // 
    // If no data runs were found, is because we were already at the end of 
    // the directory. Then Return kFALSE
    //
    if ( fDataRuns->GetNumRuns()==0 ) 
    {
	*fLog << warn << "No more Data runs left." << endl;
	return kFALSE;
    }

       
    //
    // If no new P runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fPedRuns->GetNumRuns()==0 )
    {

	*fLog << warn << "No Ped runs were found before data runs." << endl;


	// 
	// Trying to Use previous pedestal, if there were
	//
	if ( IsPreviousRunUsable(*oldPedRuns) )
	{  
	    *fLog << warn << "Using Previous Ped runs." << endl;
	    fPedRuns->Delete();
	    fPedRuns = (MRunIter*)oldPedRuns->Clone();
	}
	else
	{
	    *fLog << warn << "Previous Ped runs exists but for a different source." << endl;


	    //
	    // Found next P after D runs
	    //
	    *fLog << warn << "Looking for new Ped runs after last data run..." << endl;

	    oldDataRuns = (MRunIter*)fDataRuns->Clone();
	    oldCalRuns = (MRunIter*)fCalRuns->Clone();
    
	    Loop("P",fSrcName);
	   
	    // fDataRuns and fCalRuns has been overwriteen when calling Loop
	    fDataRuns->Delete();
	    fDataRuns = (MRunIter*)oldDataRuns->Clone();

	    fCalRuns->Delete();
	    fCalRuns = (MRunIter*)oldCalRuns->Clone();

	    
	    //
	    // Last check
	    //
	    if ( fPedRuns->GetNumRuns() == 0 )
	    {
		*fLog << err << "ERROR: Unable to found Ped runs for this source [" << fSrcName << "] ... exit." << endl;
		return kFALSE;
	    }
	}
	
    }


    //
    // If no new C runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fCalRuns->GetNumRuns()==0 )
    {
	*fLog << warn << "No Cal runs were found before data runs." << endl;

	// 
	// Trying to Use previous pedestal, if there were
	//
	if ( IsPreviousRunUsable(*oldCalRuns) )
	{
	    *fLog << warn << "Using Previous Cal runs." << endl;
	    fCalRuns->Delete();
	    fCalRuns = (MRunIter*)oldCalRuns->Clone();
	}
	else
	{

	    //
	    // Found next P after D runs
	    //
	    *fLog << warn << "Looking for new Cal runs after last data run..." << endl;
	    
	    
	    oldDataRuns = (MRunIter*)fDataRuns->Clone();
	    oldPedRuns = (MRunIter*)fPedRuns->Clone();
	        
	    Loop("C",fSrcName);
	    
	    
	    // fDataRuns and fPedRuns has been overwriteen when calling Loop
	    fDataRuns->Delete();
	    fDataRuns = (MRunIter*)oldDataRuns->Clone();
	    
	    fPedRuns->Delete();
	    fPedRuns = (MRunIter*)oldPedRuns->Clone();
	    


	    //
	    // Last check
	    //
	    if ( fCalRuns->GetNumRuns() == 0 )
	    {
		*fLog << err << "ERROR: Unable to found Cal runs for this source [" << fSrcName << "] ... exit." << endl;

		//
		// Using a default Cal run
		//
		if( fDefCalRun != 0)
		{
		    *fLog << warn << "WARN: Using a default calibration run: " << fDefCalRunPath << fDefCalRun << endl;
		    fCalRuns->AddRun(fDefCalRun,fDefCalRunPath.Data());
		}
		else
		    return kFALSE;
	    }
	}
    }




    //
    // Print the runs of this data set
    //
    Print();


    oldDataRuns->Delete();
    oldPedRuns->Delete();
    oldCalRuns->Delete();
    

    return kTRUE;
}


  
// ---------------------------------------------------------------------------
//
//  Look for a set of consecutive runs of a given type (P/C/D), starting from 
//  the last processed data run. The runs found are stored in 
//  f[Data/Ped/Cal]Runs.
//
//  If option="P" look for a set of consecutive P runs. The same for C and D 
//  runs.
//  If options="dataset", look for set of consecutive D runs, but also retrieve
//  all the P and C runs between the last processed run, and the first data 
//  run.
//
//  Method:
//  -------
//  Loop over all the files, and forechs does:
//    First, performs 2 tests
//      - run number: must be larger than the last processed data run
//      - source name: must be one of the selected source names
//    then
//      - add this run, but before, check that it correspond to the same
//        src name than the previously addded runs.
//
Int_t MDataSetIter::Loop(TString option, TString LockSrcName)
{

    //
    // Delete previous data runs
    //
    fDataRuns->Delete();
    fDataRuns = new MRunIter();

    fPedRuns->Delete();
    fPedRuns = new MRunIter();

    fCalRuns->Delete();
    fCalRuns = new MRunIter();


    //
    TString FirstSrcName = "";       // SrcName of the first run found
    Bool_t RunFound     = kFALSE; 
    Bool_t DataRunFound = kFALSE; 
    Bool_t CalRunFound  = kFALSE;
    Bool_t PedRunFound  = kFALSE;


    //
    // Check option
    //
    if ( option.CompareTo("dataset",TString::kIgnoreCase) && 
	 option.CompareTo("P",TString::kIgnoreCase) && 
	 option.CompareTo("C",TString::kIgnoreCase) && 
	 option.CompareTo("D",TString::kIgnoreCase) )
    {
	*fLog << err << "ERROR: option [" << option << "] invalid...exit" << endl;
	return kFALSE;
    }
    
    char SelectedType = !option.CompareTo("dataset",TString::kIgnoreCase) ? 'D' : option[0];
  
  


    //
    // Loop over all the files in the directory
    //
    TIter next(&fFileList);;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {  
	TString name, path, date, src;
	Int_t run;
	char type;

	ScanFileName(n->GetName(), name, path, date, src, &run, &type);

	
	// 
	// Skip until finding the following run after the fLastDataRun
	//
	if( run <= fLastProcessedRun )
	    continue;
	
	//
	// Check that the source name is among one of the selected source names
	//
	if( LockSrcName != "")
	{	
	    if( src != LockSrcName )
		continue;
	}
	else
	{
	    if( !CheckSourceName(src) )
		continue;
	}

	//
	// If a Data run has already be found, the next files has to be only
	// of type Data, if not finish.
	//
	if( !option.CompareTo("dataset",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType )
	    break;

	else if( !option.CompareTo("P",TString::kIgnoreCase) && PedRunFound==kTRUE && type!=SelectedType ) 
	    break;
	
	else if( !option.CompareTo("C",TString::kIgnoreCase) && CalRunFound==kTRUE && type!=SelectedType ) 
	    break;
	
	else if( !option.CompareTo("D",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType) 
	    break;
	
	
	
	//
	// For Checking that this run has the same name that the first one of 
	// this set
	//
	if (RunFound==kFALSE)
	{
	    RunFound = kTRUE;
	    FirstSrcName = src;
	}

	if( src != FirstSrcName)
	{
	    if(type!='C')
	    {  
		*fLog << err << "ERROR: Source Name differs inside data set ("
		  << src << " vs. " << FirstSrcName << ") ...exit." << endl;
		return kFALSE;
	    }
	    else
	    {
	    	*fLog << warn << "Warning: Cal Source Name differs inside data set (" << src << " vs. " << FirstSrcName << ")." << endl;
	    }
	}


	//
	// Add this run to its corresponding MRunIter
	//
	switch (type)
	{
	case 'D':
	    if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
	         !option.CompareTo("D",TString::kIgnoreCase) )
	    {
		*fLog << "Adding Data run: " << run << " [" << src << "]" << endl;
	
		fDataRuns->AddRun(run,path.Data());
		
		fSrcName = src;
		fDate = date;
		
		DataRunFound = kTRUE;

		fLastProcessedRun = run;
		
	    }
	    break;
	
	case 'P': 
	  if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
	       !option.CompareTo("P",TString::kIgnoreCase)  )
	    {
		*fLog << "Adding Ped run: "<< run  << " [" << src << "]" << endl;   
		
		PedRunFound = kTRUE;
		 
		fPedRuns->AddRun(run,path.Data());
	    }
	    break;

	case 'C':
	   if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
		!option.CompareTo("C",TString::kIgnoreCase)  )
	    {

	      if(CalRunFound==kFALSE)
	      {
		  *fLog << "Adding Cal run: "<< run  << " [" << src << "]" << endl;		
		  CalRunFound = kTRUE;
		    
		  fCalRuns->AddRun(run,path.Data());
	      }
	      else 
		  *fLog << "SKIPPING Cal run: "<< run  << " [" << src << "]" << endl;
	      

	    }
	    break;
	}

	
    } // End loop


    return kTRUE;
}





// ---------------------------------------------------------------------------
//
//  By default print the P,C and D runs of the current data set. With the
//  option "all" print only all the files in the FileList.
//
void MDataSetIter::Print(const Option_t *option) const
{

    //
    //  Print all the files in the FileList.
    //
    TString s(option);
    if (s.Contains("all", TString::kIgnoreCase))
    {
	fFileList.Print();
	return;
    }

    
    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();


    //
    // Print the runs of this data set
    //
    TString file;

    *fLog << all << endl << " pedestal runs: " << endl;
    *fLog << "---------------" << endl;
    while( !(file=fPedRuns->Next()).IsNull() )
	*fLog << file << endl;

   	   
    *fLog << endl << " calibration runs: " << endl; 
    *fLog << "------------------" << endl;
    while( !(file=fCalRuns->Next()).IsNull() )
	*fLog << file << endl;


    *fLog << endl << " data runs: " << endl;
    *fLog << "-----------" << endl;
    while( !(file=fDataRuns->Next()).IsNull() )
	*fLog << file << endl;
    
    
    *fLog << endl;


    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();

}


