/* ======================================================================== *\ ! ! * ! * This file is part of MARS, the MAGIC Analysis and Reconstruction ! * Software. It is distributed to you in the hope that it can be a useful ! * and timesaving tool in analysing Data of imaging Cerenkov telescopes. ! * It is distributed WITHOUT ANY WARRANTY. ! * ! * Permission to use, copy, modify and distribute this software and its ! * documentation for any purpose is hereby granted without fee, ! * provided that the above copyright notice appear in all copies and ! * that both that copyright notice and this permission notice appear ! * in supporting documentation. It is provided "as is" without express ! * or implied warranty. ! * ! ! ! Author(s): Marcos Lopez, 4/2004 ! ! Copyright: MAGIC Software Development, 2000-2004 ! ! \* ======================================================================== */ ///////////////////////////////////////////////////////////////////////////// // // MDataSetIter // // This class serves to divide all the data taking during a given night // (i.e, all the root files of a given directory), // into different data sets of files to be analyzed together. A data set is a // set of Data runs, along which their closer Ped and Cal runs. // // Usage: // ------ // - To specify the directory/ies where the data are use: // AddDirectory() // // - To specify only those files for a given source name (all the files with // a name different from the ones in the source-name-list will be ignored) // use: // SelectSourceName() // You can select several src names. // By default, if any explict source name is specified, all are valid. // // - To retrieve the next data set (Ped, Cal, and Data runs) use: // NextDataSet() // GetPefDataRuns() // GetCalRuns() // GetDatRuns() // // Two problems: // -Sources not desired : ZetaTauri // - Not P/C runs before data runs // // ///////////////////////////////////////////////////////////////////////////// #include "MDataSetIter.h" #include "MLog.h" #include "MLogManip.h" #include "MRunIter.h" #include "TObjArray.h" #include ClassImp(MDataSetIter); using namespace std; // ---------------------------------------------------------------------------- // // Default constructor. // MDataSetIter::MDataSetIter() : fLastProcessedRun(0), fLog(&gLog), fDefCalRun(0) { fPedRuns = new MRunIter; fCalRuns = new MRunIter; fDataRuns = new MRunIter; } // ---------------------------------------------------------------------------- // // Check wheather the source name is inside the list of selected source names. // If not any source name has been selected, by default, any source name is // valid. // Return: // kTRUE: this run has to be selected // kFALSE: this run has to be skipped // Int_t MDataSetIter::CheckSourceName(TString& src) { // If any source has been especified, all sorce name are accepted if(fSrcList.GetLast()==-1) return kTRUE; // // For skipping Ped and Cal runs with CL (aftet June 2004) // if(src.Contains("CL") || src.Contains("ContL")) { *fLog << warn << "Skipping CL run [" << src << "]" << endl; return kFALSE; } // // For dealing with the new calibration files nomenclature // (after 23-3-2004) of the type "CrabNebula-5blue", and for off data // like OffMrk421-1, OffMrk421-3 etc, we assume that any source name // can be made up of two parts separated by a dash: the source name and // a suffix indicated for instance the calibration colot of kind of off // data. This suffix is then ignored for checking the source name, ie., // any suffix is valid ! // if(src.Contains("-")) { *fLog << warn << "For source [" << src << "] ignoring suffix [" << src( src.First("-")+1, src.Length() ) << "]" << endl; src.Remove(src.First("-")); } // // Loop // TIter next(&fSrcList); TString ValidSrc; TNamed* n; while ( (n=(TNamed*)next()) ) { ValidSrc = n->GetName(); //if (src==ValidSrc) // For dealing with new calib files as Mrk421blue5 if (src.Contains(ValidSrc)) return kTRUE; } *fLog << warn << "Source [" << src << "] is not in the list of selected source names." << endl; return kFALSE; } // --------------------------------------------------------------------------- // // Scan a file name. // void MDataSetIter::ScanFileName(const TString& file, TString& name, TString& path, TString& date, TString& src, Int_t* run, char* type) { const Int_t slash = file.Last('/'); // Get File name and Path path = file(0,slash+1); name = file(slash+1,file.Length()); // Get date date = name(0,8); // Get run number const TString runstr = name(9,5); *run = atoi(runstr.Data()); // Get run type *type = name[15]; // Get source name src = name(17,name.Last('_')-17); } // --------------------------------------------------------------------------- // // Add all the files inside a given directory to the fFileList. // void MDataSetIter::AddToFileList(MDirIter& dir) { TString name; while (!(name=dir.Next()).IsNull()) { fFileList.Add((TObject*)(new TNamed((const char*)name, ""))); } // // Sort File List by name // fFileList.Sort(); } // ---------------------------------------------------------------------------- // // Overload MDirIter::AddDirectory // // Each time a new directory is added, all the files inside that directory // are added to the fFileList. // // Returns the number of directories added // Int_t MDataSetIter::AddDirectory(const char *dir, const char *filter, Int_t recursive) { const Int_t rc = MDirIter::AddDirectory(dir, filter, recursive); if (rc==0) return 0; // // Add the files inside that directory to the fFileList. In order to // add only the files inside the new directory added, we create // and MDirIter with only that directory. Otherwise, it we call // directly this.Next() inside FillFileList the new files and the // previously existing one will be again adde to the fFileList. // MDirIter next(dir,filter,recursive); AddToFileList(next); return kTRUE; } // ---------------------------------------------------------------------------- // // Add a source name to the list of selected source names. // void MDataSetIter::SelectSourceName(const char *src) { fSrcList.Add((TObject*)(new TNamed(src, src))); } // ---------------------------------------------------------------------------- // // First check that previous run was not empty, and then that it contains // runs for the current source (fSrcName) // Int_t MDataSetIter::IsPreviousRunUsable(MRunIter& oldRuns) { if( oldRuns.GetNumRuns() == 0) { *fLog << warn << "Doesn't exist previous set." << endl; return kFALSE; } // // Go back to ther first entry // oldRuns.Reset(); // // Check that the previous runs were for the same source name // TString name, path, date, src; Int_t run; char type; ScanFileName(oldRuns.Next(), name, path, date, src, &run, &type); // if( src != fSrcName) // { // *fLog << err << "Previous runs were for a diffrent source...exit." << endl; // return kFALSE; // } // // For dealing with calibration color we just ask than the srcname // contains the FirstSrcName // if( !src.Contains(fSrcName) ) { *fLog << warn << "Previous runs were for a diffrent source [" << src << "] vs [" << fSrcName << "] ...exit." << endl; return kFALSE; } return kTRUE; } // --------------------------------------------------------------------------- // // Assumes that a data set has the following structure: // P/C,P/C,P/C, ... // D,D,D, ... // // After finishing the loop, it checks that were found at least one Ped, one // Cal and one Data runs. // // Start from the last processed run, looking for Ped, Cal and Dat runs. As // soon a Data run is found, it start to look only for Data runs, finishing // the loop when a run of different type of Data run is found. // // Process all the runs which runnumber is later to the last processed run // (the first time this function is call fLastProcessedRun is 0, so it start // processing from the beginning) // // NOTE: This FAILS if the directory doesn't start with at lead one P and one // C runs // Int_t MDataSetIter::NextDataSet() { if(fFileList.GetLast()==-1) return kFALSE; // // Save temporaly prvious data set; // MRunIter* oldDataRuns = 0; MRunIter* oldPedRuns = 0; MRunIter* oldCalRuns = 0; if(fDataRuns) oldDataRuns = (MRunIter*)fDataRuns->Clone(); if(fPedRuns) oldPedRuns = (MRunIter*)fPedRuns->Clone(); if(fCalRuns) oldCalRuns = (MRunIter*)fCalRuns->Clone(); // // Retrieve next data set P,C and D runs // Loop("dataset"); // // If no data runs were found, is because we were already at the end of // the directory. Then Return kFALSE // if ( fDataRuns->GetNumRuns()==0 ) { *fLog << warn << "No more Data runs left." << endl; return kFALSE; } // // If no new P runs found, use the previous one, if they were for the // same source // if ( fPedRuns->GetNumRuns()==0 ) { *fLog << warn << "No Ped runs were found before data runs." << endl; // // Trying to Use previous pedestal, if there were // if ( IsPreviousRunUsable(*oldPedRuns) ) { *fLog << warn << "Using Previous Ped runs." << endl; fPedRuns->Delete(); fPedRuns = (MRunIter*)oldPedRuns->Clone(); } else { *fLog << warn << "Previous Ped runs exists but for a different source." << endl; // // Found next P after D runs // *fLog << warn << "Looking for new Ped runs after last data run..." << endl; oldDataRuns = (MRunIter*)fDataRuns->Clone(); oldCalRuns = (MRunIter*)fCalRuns->Clone(); Loop("P",fSrcName); // fDataRuns and fCalRuns has been overwriteen when calling Loop fDataRuns->Delete(); fDataRuns = (MRunIter*)oldDataRuns->Clone(); fCalRuns->Delete(); fCalRuns = (MRunIter*)oldCalRuns->Clone(); // // Last check // if ( fPedRuns->GetNumRuns() == 0 ) { *fLog << err << "ERROR: Unable to found Ped runs for this source [" << fSrcName << "] ... exit." << endl; return kFALSE; } } } // // If no new C runs found, use the previous one, if they were for the // same source // if ( fCalRuns->GetNumRuns()==0 ) { *fLog << warn << "No Cal runs were found before data runs." << endl; // // Trying to Use previous pedestal, if there were // if ( IsPreviousRunUsable(*oldCalRuns) ) { *fLog << warn << "Using Previous Cal runs." << endl; fCalRuns->Delete(); fCalRuns = (MRunIter*)oldCalRuns->Clone(); } else { // // Found next P after D runs // *fLog << warn << "Looking for new Cal runs after last data run..." << endl; oldDataRuns = (MRunIter*)fDataRuns->Clone(); oldPedRuns = (MRunIter*)fPedRuns->Clone(); Loop("C",fSrcName); // fDataRuns and fPedRuns has been overwriteen when calling Loop fDataRuns->Delete(); fDataRuns = (MRunIter*)oldDataRuns->Clone(); fPedRuns->Delete(); fPedRuns = (MRunIter*)oldPedRuns->Clone(); // // Last check // if ( fCalRuns->GetNumRuns() == 0 ) { *fLog << err << "ERROR: Unable to found Cal runs for this source [" << fSrcName << "] ... exit." << endl; // // Using a default Cal run // if( fDefCalRun != 0) { *fLog << warn << "WARN: Using a default calibration run: " << fDefCalRunPath << fDefCalRun << endl; fCalRuns->AddRun(fDefCalRun,fDefCalRunPath.Data()); } else return kFALSE; } } } // // Print the runs of this data set // Print(); oldDataRuns->Delete(); oldPedRuns->Delete(); oldCalRuns->Delete(); return kTRUE; } // --------------------------------------------------------------------------- // // Look for a set of consecutive runs of a given type (P/C/D), starting from // the last processed data run. The runs found are stored in // f[Data/Ped/Cal]Runs. // // If option="P" look for a set of consecutive P runs. The same for C and D // runs. // If options="dataset", look for set of consecutive D runs, but also retrieve // all the P and C runs between the last processed run, and the first data // run. // // Method: // ------- // Loop over all the files, and forechs does: // First, performs 2 tests // - run number: must be larger than the last processed data run // - source name: must be one of the selected source names // then // - add this run, but before, check that it correspond to the same // src name than the previously addded runs. // Int_t MDataSetIter::Loop(TString option, TString LockSrcName) { // // Delete previous data runs // fDataRuns->Delete(); fDataRuns = new MRunIter(); fPedRuns->Delete(); fPedRuns = new MRunIter(); fCalRuns->Delete(); fCalRuns = new MRunIter(); // TString FirstSrcName = ""; // SrcName of the first run found Bool_t RunFound = kFALSE; Bool_t DataRunFound = kFALSE; Bool_t CalRunFound = kFALSE; Bool_t PedRunFound = kFALSE; // // Check option // if ( option.CompareTo("dataset",TString::kIgnoreCase) && option.CompareTo("P",TString::kIgnoreCase) && option.CompareTo("C",TString::kIgnoreCase) && option.CompareTo("D",TString::kIgnoreCase) ) { *fLog << err << "ERROR: option [" << option << "] invalid...exit" << endl; return kFALSE; } char SelectedType = !option.CompareTo("dataset",TString::kIgnoreCase) ? 'D' : option[0]; // // Loop over all the files in the directory // TIter next(&fFileList);; TNamed* n; while ( (n=(TNamed*)next()) ) { TString name, path, date, src; Int_t run; char type; ScanFileName(n->GetName(), name, path, date, src, &run, &type); // // Skip until finding the following run after the fLastDataRun // if( run <= fLastProcessedRun ) continue; // // Check that the source name is among one of the selected source names // if( LockSrcName != "") { if( src != LockSrcName ) continue; } else { if( !CheckSourceName(src) ) continue; } // // If a Data run has already be found, the next files has to be only // of type Data, if not finish. // if( !option.CompareTo("dataset",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType ) break; else if( !option.CompareTo("P",TString::kIgnoreCase) && PedRunFound==kTRUE && type!=SelectedType ) break; else if( !option.CompareTo("C",TString::kIgnoreCase) && CalRunFound==kTRUE && type!=SelectedType ) break; else if( !option.CompareTo("D",TString::kIgnoreCase) && DataRunFound==kTRUE && type!=SelectedType) break; // // For Checking that this run has the same name that the first one of // this set // if (RunFound==kFALSE) { RunFound = kTRUE; FirstSrcName = src; } if( src != FirstSrcName) { if(type!='C') { *fLog << err << "ERROR: Source Name differs inside data set (" << src << " vs. " << FirstSrcName << ") ...exit." << endl; return kFALSE; } else { *fLog << warn << "Warning: Cal Source Name differs inside data set (" << src << " vs. " << FirstSrcName << ")." << endl; } } // // Add this run to its corresponding MRunIter // switch (type) { case 'D': if( !option.CompareTo("dataset",TString::kIgnoreCase) || !option.CompareTo("D",TString::kIgnoreCase) ) { *fLog << "Adding Data run: " << run << " [" << src << "]" << endl; fDataRuns->AddRun(run,path.Data()); fSrcName = src; fDate = date; DataRunFound = kTRUE; fLastProcessedRun = run; } break; case 'P': if( !option.CompareTo("dataset",TString::kIgnoreCase) || !option.CompareTo("P",TString::kIgnoreCase) ) { *fLog << "Adding Ped run: "<< run << " [" << src << "]" << endl; PedRunFound = kTRUE; fPedRuns->AddRun(run,path.Data()); } break; case 'C': if( !option.CompareTo("dataset",TString::kIgnoreCase) || !option.CompareTo("C",TString::kIgnoreCase) ) { if(CalRunFound==kFALSE) { *fLog << "Adding Cal run: "<< run << " [" << src << "]" << endl; CalRunFound = kTRUE; fCalRuns->AddRun(run,path.Data()); } else *fLog << "SKIPPING Cal run: "<< run << " [" << src << "]" << endl; } break; } } // End loop return kTRUE; } // --------------------------------------------------------------------------- // // By default print the P,C and D runs of the current data set. With the // option "all" print only all the files in the FileList. // void MDataSetIter::Print(const Option_t *option) const { // // Print all the files in the FileList. // TString s(option); if (s.Contains("all", TString::kIgnoreCase)) { fFileList.Print(); return; } // // Reset // fPedRuns->Reset(); fCalRuns->Reset(); fDataRuns->Reset(); // // Print the runs of this data set // TString file; *fLog << all << endl << " pedestal runs: " << endl; *fLog << "---------------" << endl; while( !(file=fPedRuns->Next()).IsNull() ) *fLog << file << endl; *fLog << endl << " calibration runs: " << endl; *fLog << "------------------" << endl; while( !(file=fCalRuns->Next()).IsNull() ) *fLog << file << endl; *fLog << endl << " data runs: " << endl; *fLog << "-----------" << endl; while( !(file=fDataRuns->Next()).IsNull() ) *fLog << file << endl; *fLog << endl; // // Reset // fPedRuns->Reset(); fCalRuns->Reset(); fDataRuns->Reset(); }