/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Marcos Lopez, 4/2004 <mailto:marcos@gae.ucm.es>
!
!   Copyright: MAGIC Software Development, 2000-2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MDataSetIter
//
//  This class serves to divide all the data taking during a given night
//  (i.e, all the root files of a given directory), 
//  into different data sets of files to be analyzed together. A data set is a
//  set of Data runs, along which their closer Ped and Cal runs.
//
//  Usage:
//  ------
//   - To specify the directory/ies where the data are use:
//        AddDirectory()
//
//   - To specify only those files for a given source name (all the files with
//     a name different from the ones in the source-name-list will be ignored)
//     use:
//        SelectSourceName()
//     You can select several src names.
//     By default, if any explict source name is specified, all are valid.
//
//   - To retrieve the next data set (Ped, Cal, and Data runs) use:
//        NextDataSet()
//        GetPefDataRuns()
//        GetCalRuns()
//        GetDatRuns()
//
// Two problems:
//    -Sources not desired : ZetaTauri
//    - Not P/C runs before data runs
//
//
/////////////////////////////////////////////////////////////////////////////
#include "MDataSetIter.h"

#include "MLog.h"
#include "MLogManip.h"

#include "MRunIter.h"
#include "MSequence.h"

#include "TObjArray.h"

#include <TSystem.h>
#include <fstream>

ClassImp(MDataSetIter);

using namespace std;



// ----------------------------------------------------------------------------
//
//  Default constructor.
//
MDataSetIter::MDataSetIter() 
    : fInitialRun(-1), fEndRun(-1), fLastProcessedDataRun(0), fLastDataRun(0), fDefCalRun(0), fDefCalRunPath(""), fLog(&gLog)
{
    fPedRuns = new MRunIter;
    fCalRuns = new MRunIter;
    fDataRuns = new MRunIter;
    fSequence = new MSequence;
}


// ---------------------------------------------------------------------------
//
//  Add all the files inside a given directory to the fFileList.
//
void MDataSetIter::AddToFileList(MDirIter& dir)
{
    TString name;
   
    while (!(name=dir.Next()).IsNull())
     	fFileList.Add((TObject*)(new TNamed((const char*)name, "")));
   

    //
    // Sort File List by name
    //
    fFileList.Sort();

    //
    // Find last Data runnumber of the filelist to be processed
    //
    FindLastDataRun();
}


// -------------------------------------------------------------------------
//
// Find last Data runnumber of the filelist to be processed
//
void MDataSetIter::FindLastDataRun()
{

    if(fFileList.GetLast()==-1)
	return;


    TIter next(&fFileList,kIterBackward);
    TNamed* n;

    while ( (n=(TNamed*)next()) )
    {  
	TString name, path, date, src;
	Int_t run;
	char type;
	
	ScanFileName(n->GetName(), name, path, date, src, &run, &type);

	if(type!='D')
	    continue;

	
	if(fEndRun==-1)
	{
	    if(run>fLastDataRun)
		fLastDataRun = run;
	    
	    break;
	}
	else
	{
	    if(run<=fEndRun)
	    {
		if(run>fLastDataRun)
		    fLastDataRun = run;
		
		break;
	    }
	    
	}
    }

}


// ----------------------------------------------------------------------------
//
//  Overload MDirIter::AddDirectory
//
//  Each time a new directory is added, all the files inside that directory
//  are added to the fFileList.
//
//  Returns the number of directories added
//
Int_t MDataSetIter::AddDirectory(const char *dir, const char *filter, Int_t recursive) 
{
    const Int_t rc = MDirIter::AddDirectory(dir, filter, recursive);
    if (rc==0)
	return 0;
  
    //
    // Add the files inside that directory to the fFileList. In order to 
    // add only the files inside the new directory added, we create
    // and MDirIter with only that directory. Otherwise, it we call 
    // directly this.Next() inside FillFileList the new files and the 
    // previously existing one will be again adde to the fFileList. 
    //
    MDirIter next(dir,filter,recursive);
    AddToFileList(next);

    next.Print() ;
    
    return kTRUE;
} 


// ----------------------------------------------------------------------------
//
//  Add a source name to the list of selected source names.
//
void MDataSetIter::SelectSourceName(const char *src)
{
    fSrcList.Add((TObject*)(new TNamed(src, src)));  
}
    

// ----------------------------------------------------------------------------
//
//  Check wheather the source name is inside the list of selected source names.
//  If not any source name has been selected, by default, any source name is
//  valid.
//  Return:
//   kTRUE: this run has to be selected
//   kFALSE: this run has to be skipped
//
Int_t MDataSetIter::CheckSourceName(TString& src)
{

    // If any source has been especified, all sorce name are accepted
    if(fSrcList.GetLast()==-1)
	return kTRUE;


    //
    // For skipping Ped and Cal runs with CL (aftet June 2004)
    //
    if(src.Contains("CL") || src.Contains("ContL"))
    {
        *fLog << warn << "Skipping CL run [" << src << "]" << endl;
	return kFALSE;
    }

    //
    // Skip Green and Blue calib
    //
    if(src.Contains("Blue",TString::kIgnoreCase) || src.Contains("Green",TString::kIgnoreCase))
    {
        *fLog << warn << "Skipping Blue/Green run [" << src << "]" << endl;
        return kFALSE;
    }



    //
    // For dealing with the new calibration files nomenclature 
    // (after 23-3-2004) of the type "CrabNebula-5blue", and for off data
    // like OffMrk421-1, OffMrk421-3 etc, we assume that any source name 
    // can be made up of two parts separated by a dash: the source name and 
    // a suffix indicated for instance the calibration colot of kind of off 
    // data. This suffix is then ignored for checking the source name, ie., 
    // any suffix is valid !  
    //
    if(src.Contains("-"))
    {
	//*fLog << warn << "For source [" << src << "] ignoring suffix [" 
	//     << src( src.First("-")+1, src.Length() ) << "]" << endl;
	src.Remove(src.First("-"));
    }


    //
    // Loop 
    //
    TIter next(&fSrcList);

    TString ValidSrc;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {
	ValidSrc = n->GetName();
	
	if (src==ValidSrc)       // For dealing with new calib files as Mrk421blue5
	//if (src.Contains(ValidSrc))
	    return kTRUE;
    }

    *fLog << warn << "Source [" << src << "] is not in the list of selected source names." << endl;

    return kFALSE;
}

// -------------------------------------------------------------------
//	
// Compare two source names
//
Int_t MDataSetIter::CompareSourceNames(TString& src1, TString& src2)
{
    if(src1.Contains("-"))
    {
        //*fLog << warn << "For source [" << src << "] ignoring suffix ["
        //     << src( src.First("-")+1, src.Length() ) << "]" << endl;
        src1.Remove(src1.First("-"));
    }


    if(src2.Contains("-"))
    {
        //*fLog << warn << "For source [" << src << "] ignoring suffix ["
        //     << src( src.First("-")+1, src.Length() ) << "]" << endl;
        src2.Remove(src2.First("-"));
    }

    return (src1.CompareTo(src2)==0) ? kTRUE : kFALSE;
}

// ---------------------------------------------------------------------------
//
//  Scan a file name. 
//
void MDataSetIter::ScanFileName(const TString& file, TString& name, TString& path, TString& date, TString& src, Int_t* run, char* type)
{
    const Int_t slash = file.Last('/');

    // Get File name and Path
    path = file(0,slash+1);
    name = file(slash+1,file.Length());
  
    TString tmp = name;
    

    // Get date
    date = tmp(0,tmp.First('_'));
    tmp = tmp.Remove(0,tmp.First('_')+1);

 
    // Get run number
    const TString runstr = tmp(0,tmp.First('_')); 
    *run = atoi(runstr.Data());
    tmp = tmp.Remove(0,tmp.First('_')+1);


    // Get run type
    *type = tmp[0];
    tmp = tmp.Remove(0,tmp.First('_')+1);
  

    // Get source name
    //src = tmp(0,tmp.First('_')); 
    //
    // To deal with  4C15 with filemames doesn't include '_E' (18-11-2004) 
    //
    if (tmp.Contains('_'))
	src = tmp(0,tmp.First('_'));
    else	
	src = tmp(0,tmp.First('.'));
}


// ----------------------------------------------------------------------------
//
// First check that previous run was not empty, and then that it contains
// runs for the current source (fSrcName)
//
Int_t MDataSetIter::IsPreviousRunUsable(MRunIter& oldRuns)
{
    if( oldRuns.GetNumRuns() == 0)
    {
	*fLog << warn << "Doesn't exist previous set." << endl;
	return kFALSE;
    }
    
    //
    // Go back to ther first entry
    //	
    oldRuns.Reset(); 
	
    //
    // Check that the previous runs were for the same source name
    //
    TString name, path, date, src;
    Int_t run;
    char type;
    
    ScanFileName(oldRuns.Next(), name, path, date, src, &run, &type);
 
    
    if ( !CompareSourceNames(src,fSrcName) )
    //
    // For dealing with calibration color we just ask than the srcname 
    // contains the FirstSrcName
    //
    //if( !src.Contains(fSrcName) )
    {
	*fLog << warn << "Previous runs were for a diffrent source [" << src << "] vs [" << fSrcName << "] ...exit." << endl;
	return kFALSE;
    }
    
    
    return kTRUE;
}
    

// ---------------------------------------------------------------------------
// 
//  Assumes that a data set has the following structure:
//    P/C,P/C,P/C, ...
//    D,D,D, ...  
// 
//  After finishing the loop, it checks that were found at least one Ped, one
//  Cal and one Data runs. 
//
//  Start from the last processed run, looking for Ped, Cal and Dat runs. As
//  soon a Data run is found, it start to look only for Data runs, finishing 
//  the loop when a run of different type of Data run is found.
// 
// Process all the runs which runnumber is later to the last processed run 
//  (the first time this function is call fLastProcessedDataRun is 0, so it start
//  processing from the beginning)
//
//  NOTE: This FAILS if the directory doesn't start with at lead one P and one 
//        C runs  
//
Int_t MDataSetIter::NextDataSet()
{
   
    if(fFileList.GetLast()==-1)
    {
	*fLog << warn << "File list empty." << endl;
	return kFALSE;    
    }


    //
    // Save temporaly prvious data set;
    //
    MRunIter* oldDataRuns = 0;
    MRunIter* oldPedRuns = 0;
    MRunIter* oldCalRuns = 0;
    
    if(fDataRuns)
      oldDataRuns = (MRunIter*)fDataRuns->Clone();
    if(fPedRuns)
      oldPedRuns = (MRunIter*)fPedRuns->Clone();
    if(fCalRuns)
      oldCalRuns = (MRunIter*)fCalRuns->Clone();
    

  
    //
    // Retrieve next data set P,C and D runs
    //
    Loop("dataset");


    // 
    // If no data runs were found, is because we were already at the end of 
    // the directory. Then Return kFALSE
    //
    if ( fDataRuns->GetNumRuns()==0 )
    {
	*fLog << warn << "No more Data runs left." << endl;
	return kFALSE;
    }

       
    //
    // If no new P runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fPedRuns->GetNumRuns()==0 )
    {

	*fLog << warn << "No Ped runs were found before data runs." << endl;


	// 
	// Trying to Use previous pedestal, if there were
	//
	if ( IsPreviousRunUsable(*oldPedRuns) )
	{  
	    *fLog << warn << "Using Previous Ped runs." << endl;
	    fPedRuns->Delete();
	    fPedRuns = (MRunIter*)oldPedRuns->Clone();
	}
	else
	{
	    *fLog << warn << "Previous Ped runs exists but for a different source." << endl;


	    //
	    // Found next P after D runs
	    //
	    *fLog << warn << "Looking for new Ped runs after last data run..." << endl;

	    oldDataRuns = (MRunIter*)fDataRuns->Clone();
	    oldCalRuns = (MRunIter*)fCalRuns->Clone();
    
	    Loop("P",fSrcName);
	   
	    // fDataRuns and fCalRuns has been overwriteen when calling Loop
	    fDataRuns->Delete();
	    fDataRuns = (MRunIter*)oldDataRuns->Clone();

	    fCalRuns->Delete();
	    fCalRuns = (MRunIter*)oldCalRuns->Clone();

	    
	    //
	    // Last check
	    //
	    if ( fPedRuns->GetNumRuns() == 0 )
	    {
		*fLog << err << "ERROR: Unable to found Ped runs for this source [" << fSrcName << "] ... exit." << endl;
		return kFALSE;
	    }
	}
	
    }


    //
    // If no new C runs found, use the previous one, if they were for the 
    // same source
    //
    if ( fCalRuns->GetNumRuns()==0 )
    {
	*fLog << warn << "No Cal runs were found before data runs." << endl;

	// 
	// Trying to Use previous pedestal, if there were
	//
	if ( IsPreviousRunUsable(*oldCalRuns) )
	{
	    *fLog << warn << "Using Previous Cal runs." << endl;
	    fCalRuns->Delete();
	    fCalRuns = (MRunIter*)oldCalRuns->Clone();
	}
	else
	{

	    //
	    // Found next P after D runs
	    //
	    *fLog << warn << "Looking for new Cal runs after last data run..." << endl;
	    
	    
	    oldDataRuns = (MRunIter*)fDataRuns->Clone();
	    oldPedRuns = (MRunIter*)fPedRuns->Clone();
	        
	    Loop("C",fSrcName);
	    
	    
	    // fDataRuns and fPedRuns has been overwriteen when calling Loop
	    fDataRuns->Delete();
	    fDataRuns = (MRunIter*)oldDataRuns->Clone();
	    
	    fPedRuns->Delete();
	    fPedRuns = (MRunIter*)oldPedRuns->Clone();
	    


	    //
	    // Last check
	    //
	    if ( fCalRuns->GetNumRuns() == 0 )
	    {
		*fLog << err << "ERROR: Unable to found Cal runs for this source [" << fSrcName << "] ... exit." << endl;

		//
		// Using a default Cal run
		//
		if( fDefCalRun != 0)
		{
		    *fLog << warn << "WARN: Using a default calibration run: " << fDefCalRunPath << fDefCalRun << endl;
		    fCalRuns->AddRun(fDefCalRun,fDefCalRunPath.Data());
		}
		else
		    return kFALSE;
	    }
	}
    }




    //
    // Print the runs of this data set
    //
    //Print();


    oldDataRuns->Delete();
    oldPedRuns->Delete();
    oldCalRuns->Delete();
    

    return kTRUE;
}


// -----------------------------------------------------------------------
//
//  Write an ASCII file with the sequence info. 
//  The name of the file is 'sequenceRunNum.txt' where 
//  RunNum is the run number of the first data file to analyse. We use 
//  the data runnumber instead of calib run because several sequences can
//  have the same calib run (just the P can change)
//
Int_t MDataSetIter::WriteSequence()
{
    //
    // Sequence
    //
    //fSequence->SetupDatRuns(fDataRuns, fPathData, "D");
    //fSequence->SetupPedRuns(fPedRuns, fPathData, "P");
    //fSequence->SetupCalRuns(fCalRuns, fPathData, "C");
    
    //fSequence->Print();

    TString sped =  fPedRuns->GetRunsAsFileName();
    TString scal =  fCalRuns->GetRunsAsFileName();
    TString sdata = fDataRuns->GetRunsAsFileName();

    sped.ReplaceAll("_"," ");
    scal.ReplaceAll("_"," ");
    sdata.ReplaceAll("_"," ");

   
    Int_t seqnum;
    //sscanf(scal.Data(),"%d",&seqnum);
    sscanf(sdata.Data(),"%d",&seqnum);

    TString fname = Form("sequence_%s_%d.txt",fDate.Data(), seqnum);
    ofstream out(fname);


    TString date = fDate;
    date.Insert(4,"-");
    date.Insert(7,"-");


    //out << "# Sequence number (identifier)" <<endl;
    out << "Sequence: " << seqnum << endl;
    //out << endl;
    //out << "# date of sunrise of the observation night" << endl;
    //out << "Night: 2004-09-21" << endl;
    out << "Night: " << date << endl;
    out << endl;
    //out << "# Source name of all runs of sequence" << endl;
    //out << "Source: " << fSrcName << endl;
    //out << endl;
    //out << "# List of all runs of this sequence" << endl;
    //out << "Runs: " << scal << " " << sped << " "<< sdata << endl;  
    //out << endl;
    //out << "# List of all calibration runs of this sequence" << endl;
    out << "CalRuns: " << scal << endl;
    //out << "# List of all pedestal runs of this sequence" << endl;
    out << "PedRuns: " << sped << endl;
    //out << "# List of all data runs of this sequence" << endl;
    out << "DatRuns: "<<  sdata << endl;
    out.close();

    
    return kTRUE;
}


  
// ---------------------------------------------------------------------------
//
//  Look for a set of consecutive runs of a given type (P/C/D), starting from 
//  the last processed data run. The runs found are stored in 
//  f[Data/Ped/Cal]Runs.
//
//  If option="P" look for a set of consecutive P runs. The same for C and D 
//  runs.
//  If options="dataset", look for set of consecutive D runs, but also retrieve
//  all the P and C runs between the last processed run, and the first data 
//  run.
//
//  Method:
//  -------
//  Loop over all the files, and forechs does:
//    First, performs 2 tests
//      - run number: must be larger than the last processed data run
//      - source name: must be one of the selected source names
//    then
//      - add this run, but before, check that it correspond to the same
//        src name than the previously addded runs.
//
Int_t MDataSetIter::Loop(TString option, TString LockSrcName)
{

    //
    // Delete previous data runs
    //
    fDataRuns->Delete();
    fDataRuns = new MRunIter();

    fPedRuns->Delete();
    fPedRuns = new MRunIter();

    fCalRuns->Delete();
    fCalRuns = new MRunIter();

    //
    // Init
    //
    TString FirstSrcName = "";     // SrcName of the first run found
    Bool_t RunFound     = kFALSE; 
    Bool_t DataRunFound = kFALSE; 
    Bool_t CalRunFound  = kFALSE;
    Bool_t PedRunFound  = kFALSE;

   


    //
    // Check option
    //
    if ( option.CompareTo("dataset",TString::kIgnoreCase) && 
	 option.CompareTo("P",TString::kIgnoreCase) && 
	 option.CompareTo("C",TString::kIgnoreCase) && 
	 option.CompareTo("D",TString::kIgnoreCase) )
    {
	*fLog << err << "ERROR: option [" << option << "] invalid...exit" << endl;
	return kFALSE;
    }
    
    char SelectedType = !option.CompareTo("dataset",TString::kIgnoreCase) ? 'D' : option[0];
  
  


    //
    // Loop over all the files in the directory
    //
    TIter next(&fFileList);;
    TNamed* n;
    while ( (n=(TNamed*)next()) )
    {  
	TString name, path, date, src;
	Int_t run;
	char type;

	ScanFileName(n->GetName(), name, path, date, src, &run, &type);


	//
	// Check that runnumber is inside the desire range of range to analyse [fInitialRun,fEndRun]
	//
	if(fEndRun !=-1 && run > fEndRun)
	    break;
	if(run < fInitialRun)
	    continue;


	// 
	// Skip until finding the following run after the last processed one, fLastDataRun
	//
	if( run <= fLastProcessedDataRun )
	    continue;

	//
	// Check that the source name is among one of the selected source names
	//
	if( LockSrcName != "")
	{	
	    FirstSrcName = LockSrcName;
	}
	
	if( !CheckSourceName(src) )
	    continue;


	//
	// If a Data run has already be found, the next files has to be only
	// of type Data, if not finish.
	//
	if( !option.CompareTo("dataset",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType )
	    break;

	else if( !option.CompareTo("P",TString::kIgnoreCase) && PedRunFound==kTRUE && type!=SelectedType ) 
	    break;
	
	else if( !option.CompareTo("C",TString::kIgnoreCase) && CalRunFound==kTRUE && type!=SelectedType ) 
	    break;
	
	else if( !option.CompareTo("D",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType) 
	    break;
	
	
	
	//
	// For Checking that this run has the same name that the first one of 
	// this set
	//
	if (RunFound==kFALSE)
	{
	    RunFound = kTRUE;
	    FirstSrcName = src;
	}

	
	if( !CompareSourceNames(src,FirstSrcName) )
	{
	    *fLog << err << "ERROR: Source Name differs inside data set ("
		  << src << " vs. " << FirstSrcName << ") ...exit." << endl;
	    return kFALSE;
	}



	//
	// Add this run to its corresponding MRunIter
	//
	switch (type)
	{
	case 'D':
	    if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
	         !option.CompareTo("D",TString::kIgnoreCase) )
	    {
		*fLog << "Adding Data run: " << run << " [" << src << "]" << endl;
	
		fDataRuns->AddRun(run,path.Data());
		
		fSrcName = src;
		fDate = date;
		
		DataRunFound = kTRUE;

		fLastProcessedDataRun = run;	
	    }
	    break;
	
	case 'P': 
	  if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
	       !option.CompareTo("P",TString::kIgnoreCase)  )
	    {
		*fLog << "Adding Ped run: "<< run  << " [" << src << "]" << endl;   
		
		PedRunFound = kTRUE;
		 
		fPedRuns->AddRun(run,path.Data());
	    }
	    break;

	case 'C':
	   if(  !option.CompareTo("dataset",TString::kIgnoreCase) || 
		!option.CompareTo("C",TString::kIgnoreCase)  )
	    {

	      if(CalRunFound==kFALSE)
	      {
		  *fLog << "Adding Cal run: "<< run  << " [" << src << "]" << endl;		
		  CalRunFound = kTRUE;
		    
		  fCalRuns->AddRun(run,path.Data());
	      }
	      else 
		  *fLog << "SKIPPING Cal run: "<< run  << " [" << src << "]" << endl;
	      

	    }
	    break;
	}

	
    } // End loop


    return kTRUE;
}





// ---------------------------------------------------------------------------
//
//  By default print the P,C and D runs of the current data set. With the
//  option "all" print only all the files in the FileList.
//
void MDataSetIter::Print(const Option_t *option) const
{

    //
    //  Print all the files in the FileList.
    //
    TString s(option);
    if (s.Contains("all", TString::kIgnoreCase))
    {
	fFileList.Print();
	return;
    }

    
    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();


    //
    // Print the runs of this data set
    //
    TString file;

    *fLog << all << endl << " pedestal runs: " << endl;
    *fLog << "---------------" << endl;
    while( !(file=fPedRuns->Next()).IsNull() )
	*fLog << file << endl;

   	   
    *fLog << endl << " calibration runs: " << endl; 
    *fLog << "------------------" << endl;
    while( !(file=fCalRuns->Next()).IsNull() )
	*fLog << file << endl;


    *fLog << endl << " data runs: " << endl;
    *fLog << "-----------" << endl;
    while( !(file=fDataRuns->Next()).IsNull() )
	*fLog << file << endl;
    
    
    *fLog << endl;


    //
    // Reset
    //
    fPedRuns->Reset(); 
    fCalRuns->Reset();
    fDataRuns->Reset();

}


// ------------------------------------------------------------------------------------
//
//  Deterimines wheather of data runs has been processed or not
//
Bool_t MDataSetIter::HasFinishedSuccesfully()
{

    if(fEndRun==-1)
	*fLog << "LastDataRun = " << fLastDataRun << endl;
    else 
	*fLog << "EndRun = " <<  fEndRun << endl;
    
    *fLog << "LastProcessedDataRun = " << fLastProcessedDataRun << endl;   


    if(fEndRun==-1)
	return fLastProcessedDataRun==fLastDataRun ? kTRUE : kFALSE;
    else 
	return fLastProcessedDataRun==fEndRun ? kTRUE : kFALSE;
       
}