Index: /trunk/MagicSoft/Mars/Changelog =================================================================== --- /trunk/MagicSoft/Mars/Changelog (revision 9017) +++ /trunk/MagicSoft/Mars/Changelog (revision 9018) @@ -43,4 +43,14 @@ * mbase/MTime.cc: - made SetSqlDateTime more flexible + + * mjobs/MJMerpp.[h,cc]: + - added (basically the code from merpp.cc) + - enhanced with the possibility to merpp a whole sequence + + * merpp.cc: + - use new MJMerpp + + * mjobs/Makefile, mjobs/JobsLinkDef.h: + - added MJMerpp Index: /trunk/MagicSoft/Mars/datacenter/macros/checkfileavail.C =================================================================== --- /trunk/MagicSoft/Mars/datacenter/macros/checkfileavail.C (revision 9017) +++ /trunk/MagicSoft/Mars/datacenter/macros/checkfileavail.C (revision 9018) @@ -18,5 +18,5 @@ ! Author(s): Daniela Dorner, 01/2005 ! -! Copyright: MAGIC Software Development, 2000-2006 +! Copyright: MAGIC Software Development, 2000-2008 ! ! @@ -34,10 +34,10 @@ // // executing the macro: -// .x checkfileavail.C+("sequencefile") +// .x checkfileavail.C+(100002) // the sequencefile (including path) has to be given, as the macro retrieves // from there the runnumbers // -// the macro returns 0, if there's no connection to the database, 2, if a -// file is missing, and 1 if all files are there +// the macro returns 0, if there's no connection to the database, 2 if a +// file is missing, and 1 if all files are there, 3 in case of error. // the return value is checked by the script, that executes the macro // @@ -45,50 +45,17 @@ // ///////////////////////////////////////////////////////////////////////////// - #include #include -#include -#include - -#include #include #include +#include "MSQLServer.h" + using namespace std; - -//check the value of a column for a run -//if a file is not available, the column contains nothing (NULL) - this value is returned -TString GetStatus(MSQLServer &serv, TEnv &rc, TString primary, TString table, TString column) +int checkfileavail(Int_t num, Int_t tel) { - TString query(Form("SELECT %s FROM %s WHERE %s=%s", - column.Data(), table.Data(), - rc.GetValue(table+".Primary", ""), - primary.Data())); - - cout << "Query: " << query << endl; - - TSQLResult *res = serv.Query(query); - if (!res) - { - cout << "Error - no run to check" << endl; - return ""; - } - - TSQLRow *row = res->Next(); - - TString ret = row ? (*row)[0] : ""; - - delete res; - - return ret; -} - -int checkfileavail(TString sequencefile) -{ - TEnv env("sql.rc"); - - MSQLServer serv(env); + MSQLServer serv("sql.rc"); if (!serv.IsConnected()) { @@ -96,4 +63,5 @@ return 0; } + cout << "checkfileavail" << endl; cout << "--------------" << endl; @@ -101,51 +69,46 @@ cout << "Connected to " << serv.GetName() << endl; cout << endl; + cout << "Sequence: " << num << endl; + cout << "Telecope: " << tel << endl; + cout << endl; - TEnv rc("steps.rc"); + // ------------------------------------------- - //reading runnumbers from Sequencefile - TEnv sequ(sequencefile); - cout << "sequ file: " << sequencefile.Data() << endl; - TString runs; - runs = sequ.GetValue("Runs", ""); - runs.ReplaceAll(" ", ","); - if (runs.IsNull()) - { - cout << "ERROR - No runs in file " << sequencefile << " found." << endl; - return 0; - } - - //getting runnumbers from database - //(not neccessary anymore -> can be changed) - TString query="SELECT fRunNumber FROM RunData WHERE fRunNumber in ("; - query +=runs.Data(); - query +=")"; + TString query = "MIN( NOT (" + "ISNULL(fRawFileAvail) OR " + "ISNULL(fCCFileAvail) OR " + "ISNULL(fCaCoFileAvail) OR " + "ISNULL(fCaCoFileFound) OR " + "ISNULL(fTimingCorrection) OR " + "ISNULL(fCompmux) " + ")) " + "FROM RunProcessStatus " + "LEFT JOIN RunData USING (fTelescopeNumber,fRunNumber,fFileNumber) " + query += Form("WHERE fSequenceFirst=%d AND fTelescopeNumber=%d", num, tel); TSQLResult *res = serv.Query(query); if (!res) - cout << "Error - no run to check" << endl; + return 3; - //check for each run the availability of files from the table RunProcessStatus - //the following columns have to be checked: - //fCCFileAvail, fCaCoFileAvail, fCaCoFileFound, fRawFileAvail, fTimingCorrection, fCompmux - //fTimingCorrection has to be checked, as only rawfiles with correct timing should be calibrated - TSQLRow *row=0; - while ((row = res->Next())) + if (res->GetRowCount()!=1) { - TString runno=(*row)[0]; - cout << "run#: " << runno << endl; - //if one value returns "" (i.e. column is NULL), 0 is returned - if (GetStatus(serv, rc, runno, "RunProcessStatus", "fCCFileAvail").IsNull() - || GetStatus(serv, rc, runno, "RunProcessStatus", "fCaCoFileAvail").IsNull() - || GetStatus(serv, rc, runno, "RunProcessStatus", "fCaCoFileFound").IsNull() - || GetStatus(serv, rc, runno, "RunProcessStatus", "fTimingCorrection").IsNull() - || GetStatus(serv, rc, runno, "RunProcessStatus", "fCompmux").IsNull() -// || GetStatus(serv, rc, runno, "RunProcessStatus", "fTimingCorrection")=="1970-01-01 00:00:00" - || GetStatus(serv, rc, runno, "RunProcessStatus", "fRawFileAvail").IsNull()) - return 2; + cout << "ERROR - Unexpected number of returned rows (" << res->GetRowCount() << ")" << endl; + delete res; + return 3; } - //if all values are okay (i.e. files are availabel), 1 is returned - return 1; + TSQlRow *row = res->Next(); + + if (!row || !(*row)[0]) + { + cout << "ERROR - Unexpected result." << endl; + delete res; + return 3; + } + + const Int_t rc = atoi((*row)[0]); + + delete res; + + return rc==1 ? 1 : 2; } - Index: /trunk/MagicSoft/Mars/datacenter/macros/checkstardone.C =================================================================== --- /trunk/MagicSoft/Mars/datacenter/macros/checkstardone.C (revision 9017) +++ /trunk/MagicSoft/Mars/datacenter/macros/checkstardone.C (revision 9018) @@ -18,5 +18,5 @@ ! Author(s): Daniela Dorner, 05/2005 ! -! Copyright: MAGIC Software Development, 2000-2006 +! Copyright: MAGIC Software Development, 2000-2008 ! ! @@ -28,5 +28,5 @@ // =============== // -// check the availability of the star files for one sequence: +// check the availability of all star files for one dataset: // if star has been executed successfully, this information has been inserted // into the database (SequenceProcessStatus) with this macro this information @@ -34,58 +34,26 @@ // // executing the macro: -// .x checkstardone.C+("sequencenumber") +// .x checkstardone.C+(1) // -// the macro returns 0, if there's no connection to the database, 2, if a -// file is missing, and 1 if all files are there +// The macro returns 0, if there's no connection to the database, 2 if +// star is not done, and 1 if star done, 3 in case of error. // the return value is checked by the script, that executes the macro // -// this macro is very similar to the macro checkfileavail.C +// This macro is very similar to the macro checkfileavail.C // ///////////////////////////////////////////////////////////////////////////// - #include #include -#include -#include - -#include #include #include +#include "MSQLServer.h" + using namespace std; - -//check the value of a column for a sequence -TString GetStatus(MSQLServer &serv, TEnv &env, TString primary, TString table, TString column) +int checkstardone(Int_t num) { - TString query(Form("SELECT %s FROM %s WHERE %s=%s", - column.Data(), table.Data(), - env.GetValue(table+".Primary", ""), - primary.Data())); - - cout << "Query: " << query << endl; - - TSQLResult *res = serv.Query(query); - if (!res) - { - cout << "Error - no sequence to check" << endl; - return ""; - } - - TSQLRow *row = res->Next(); - - TString rc = row ? (*row)[0] : ""; - - delete res; - - return rc; -} - -int checkstardone(TString datasetno) -{ - TEnv env("sql.rc"); - - MSQLServer serv(env); + MSQLServer serv("sql.rc"); if (!serv.IsConnected()) { @@ -93,4 +61,5 @@ return 0; } + cout << "checkstardone" << endl; cout << "-------------" << endl; @@ -98,12 +67,37 @@ cout << "Connected to " << serv.GetName() << endl; cout << endl; + cout << "Dataset: " << num << endl; + cout << endl; - TEnv rc("steps.rc"); + // ------------------------------------------- - //check if fStar not NULL for sequence - if (GetStatus(serv, rc, datasetno, "SequenceProcessStatus", "fStar").IsNull()) - return 2; + TString query = "NOT ISNULL(fStar) FROM SequenceProcessStatus " + "LEFT JOIN DataSetSequenceMapping USING (fTelescopeNumber,fSequenceFirst) "; + query += Form("WHERE fDataSetNumber=%d", num); - return 1; + TSQLResult *res = serv.Query(query); + if (!res) + return 3; + + if (res->GetRowCount()!=1) + { + cout << "ERROR - Unexpected number of returned rows (" << res->GetRowCount() << ")" << endl; + delete res; + return 3; + } + + TSQlRow *row = res->Next(); + + if (!row || !(*row)[0]) + { + cout << "ERROR - Unexpected result." << endl; + delete res; + return 3; + } + + const Int_t rc = atoi((*row)[0]); + + delete res; + + return rc==1 ? 1 : 2; } - Index: /trunk/MagicSoft/Mars/datacenter/scripts/checkstardone =================================================================== --- /trunk/MagicSoft/Mars/datacenter/scripts/checkstardone (revision 9017) +++ /trunk/MagicSoft/Mars/datacenter/scripts/checkstardone (revision 9018) @@ -20,5 +20,5 @@ # Author(s): Daniela Dorner 08/2004 # -# Copyright: MAGIC Software Development, 2000-2007 +# Copyright: MAGIC Software Development, 2000-2008 # # @@ -35,5 +35,4 @@ # the function setstatus. # - source `dirname $0`/sourcefile printprocesslog "INFO starting $0" @@ -64,57 +63,44 @@ for dataset in ${datasets[@]} do - echo "checking files for dataset $dataset..." >> $scriptlog 2>&1 - printprocesslog "INFO checking files for dataset $dataset" + echo "checking star done for sequences of dataset $dataset..." >> $scriptlog 2>&1 + printprocesslog "INFO checking star done for sequences of dataset $dataset" + no=`printf %08d $dataset | cut -c 0-5` no2=`printf %08d $dataset` - datasetfile=$datasetpath/$no/dataset`printf %08d $dataset`.txt - - sequences=(`cat $datasetfile | grep Sequences | sed -e 's/SequencesOn://g' | sed -e 's/SequencesOff://g'`) - if [ "$sequences" = "" ] - then - echo "no sequences found" >> $scriptlog 2>&1 - echo "continue with next dataset" >> $scriptlog 2>&1 - continue - fi - echo "sequences: "${sequences[@]} >> $scriptlog 2>&1 - + dsnum=`printf %08d $dataset` + + echo "run $program for sequence $sequence..." >> $scriptlog 2>&1 + outpath=$logpath/$program/$no/$no2 makedir $outpath >> $scriptlog 2>&1 - + primvar=$dataset setstatus "start" >> $scriptlog 2>&1 - for sequence in ${sequences[@]} - do - echo "run $program for sequence $sequence..." >> $scriptlog 2>&1 - printprocesslog "INFO check availability for sequence $sequence" + check1=`root -q -b $macrospath/checkstardone.C+\(\""$dsnum\""\) | tee $outpath/checkstardone-$dsnum.log | intgrep` - check1=`root -q -b $macrospath/checkstardone.C+\(\""$sequence\""\) | tee $outpath/checkstardone-$sequence.log | intgrep` + case $check1 in + 1) echo " check1=$check1 -> everything ok -> continue with next dataset..." >> $scriptlog 2>&1 ;; + 2) echo " check1=$check1 -> star files for dataset $dsnum not yet available -> continue..." >> $scriptlog 2>&1 + printprocesslog "INFO files not yet available for dataset $dsnum" + check="no" + break + ;; + 0) echo " check1=$check1 -> no connection to db -> continue..." >> $scriptlog 2>&1 + printprocesslog "WARN connection to DB failed" + check="no" + break + ;; + *) echo " check1=$check1 -> ERROR " >> $scriptlog 2>&1 + printprocesslog "ERROR checkstardone.C failed" + com=$Fstardone + comadd=$sequence + check=$check1 + break + ;; + esac - case $check1 in - 1) echo " check1=$check1 -> everything ok -> continue with next sequence..." >> $scriptlog 2>&1 ;; - 2) echo " check1=$check1 -> files for sequence $sequence not yet available -> continue..." >> $scriptlog 2>&1 - printprocesslog "INFO files not yet available for sequence $sequence" - check="no" - break - ;; - 0) echo " check1=$check1 -> no connection to db -> continue..." >> $scriptlog 2>&1 - printprocesslog "WARN connection to DB failed" - check="no" - break - ;; - *) echo " check1=$check1 -> ERROR " >> $scriptlog 2>&1 - printprocesslog "ERROR $program.C failed" - com=$Fstardone - comadd=$sequence - check=$check1 - break - ;; - esac - done - setstatus "stop" >> $scriptlog 2>&1 done finish >> $scriptlog 2>&1 - Index: /trunk/MagicSoft/Mars/merpp.cc =================================================================== --- /trunk/MagicSoft/Mars/merpp.cc (revision 9017) +++ /trunk/MagicSoft/Mars/merpp.cc (revision 9018) @@ -1,22 +1,11 @@ -#include -#include - -#include "MParList.h" -#include "MTaskList.h" -#include "MEvtLoop.h" - -#include "MRawFileRead.h" -#include "MReportFileReadCC.h" -#include "MWriteRootFile.h" +#include + +#include "MArgs.h" +#include "MArray.h" #include "MLog.h" #include "MLogManip.h" -#include "MArgs.h" -#include "MTime.h" -#include "MArray.h" -#include "MRawRunHeader.h" - -#include "MFDataPhrase.h" +#include "MJMerpp.h" using namespace std; @@ -57,32 +46,52 @@ gLog << all << endl; gLog << "Sorry the usage is:" << endl; - gLog << " merpp [options] inputfile[.rep,[.raw.gz],[.txt]] [outputfile[.root]]" << endl << endl; + gLog << " merpp [options] [tel:]seqnumber [outpath [inpath]]" << endl; + gLog << " merpp [options] [tel:]seqnumber outpath sumfile.txt" << endl; + gLog << " merpp [options] seqfile.txt [outpath [inpath]]" << endl; + gLog << " merpp [options] seqfile.txt outpath sumfile.txt" << endl; +// gLog << " merpp [options] mysql [outpath [inpath]]" << endl; +// gLog << " merpp [options] mysql outpath sumfile.txt" << endl; + gLog << " merpp [options] infile.raw.[gz] [outfile.root]" << endl; + gLog << " merpp [options] infile.rep [outfile.root]" << endl; + gLog << " merpp [options] infile.txt outfile.root" << endl; + gLog << endl; gLog << " Arguments:" << endl; - gLog << " inputfile.raw[.gz] Magic DAQ binary file." << endl; - gLog << " inputfile.rep Magic Central Control report file." << endl; - gLog << " inputfile.txt Magic DC currents file." << endl; - gLog << " ouputfile.root Merpped root file." << endl << endl; + gLog << " [tel:]seqnumber Telescope and sequence number to mbe merpped." << endl; + gLog << " inpath The input files whether the CC files are." << endl; + gLog << " outpath The output path where the outputfile are or stored." << endl; + gLog << " sumfile.txt A central control report summary file (--summary can be omitted)." << endl; + gLog << " infile.raw[.gz] Magic DAQ binary file." << endl; + gLog << " infile.rep Magic Central Control report file." << endl; + gLog << " infile.txt Magic DC currents file." << endl; + gLog << " outfile.root Merpped root file." << endl; +// gLog << " mysql mysql://user:password@host/database/tel:sequence" << endl; + gLog << endl; gLog << " Options:" << endl; gLog.Usage(); - gLog << " --version, -V Show startup message with version number" << endl; - gLog << " -?, -h, --help This help" << endl << endl; +// gLog << " --debug-env=0 Disable debugging setting resources " << endl; +// gLog << " --debug-env[=1] Display untouched resources after program execution" << endl; +// gLog << " --debug-env=2 Display untouched resources after eventloop setup" << endl; +// gLog << " --debug-env=3 Debug setting resources from resource file and command line" << endl; + gLog << " --debug-mem Debug memory usage" << endl << endl; + gLog << endl; gLog << " File Options:" << endl; gLog << " -c# Compression level #=1..9 [default=2]" << endl; gLog << " -f Force overwrite of an existing file" << endl; gLog << " -u, --update Update an existing file." << endl; - gLog << " --only=Name Read only reports described by MReportName. See the" << endl; - gLog << " mreport-directory for available classes." << endl << endl; + gLog << endl; gLog << " Raw Data Options:" << endl; gLog << " -ff Force merpp to ignore broken events and don't stop" << endl; - gLog << " --interleave=# Process only each i-th event [default=1]" << endl << endl; -// gLog << " --sql=mysql://user:password@url Insert run into database" << endl << endl; - gLog << " Report File Options:" << endl; + gLog << " --interleave=# Process only each i-th event [default=1]" << endl; + gLog << endl; + gLog << " Report/Currents File Options:" << endl; gLog << " --auto-time-start Take time automatically from MRawRunHeader" << endl; - gLog << " (overwrites --start=)" << endl; + gLog << " (overwrites --start=, update only)" << endl; gLog << " --auto-time-stop Take time automatically from MRawRunHeader" << endl; - gLog << " (overwrites --stop=)" << endl; + gLog << " (overwrites --stop=, update only)" << endl; gLog << " --auto-time Abbrev. for --auto-time-start and auto-time-stop" << endl; - gLog << " --start=date/time Start event time" << endl; - gLog << " --stop=date/time Stop event time" << endl; + gLog << " --start=\"time\" Start event time (format see MTime::SetSqlDateTime, update only)" << endl; + gLog << " --stop=\"time\" Stop event time (format see MTime::SetSqlDateTime, update only)" << endl; + gLog << endl; + gLog << " Report Options:" << endl; gLog << " --rep-run=# Only data corresponding to this run number as" << endl; gLog << " taken from the RUN-REPORT is extracted" << endl; @@ -98,4 +107,10 @@ gLog << " (from .rep header)" << endl; gLog << " --allfiles Don't check file type " << endl << endl; + gLog << " --only=Name Read only reports described by MReportName. See the" << endl; + gLog << " mreport-directory for available classes." << endl; + gLog << endl; + gLog << " --version, -V Show startup message with version number" << endl; + gLog << " -?, -h, --help This help" << endl; + gLog << endl; gLog << " Compatibility (deprecated):" << endl; gLog << " --run=# See --rep-run (overwritten by --rep-run)" << endl; @@ -105,54 +120,9 @@ } -// FIXME: Move to MTime (maybe 'InterpreteCmdline') -MTime AnalyseTime(TString str) -{ - Int_t y=0, ms=0, mon=0, d=0, h=0, m=0, s=0; - - const Int_t n = sscanf(str.Data(), "%d-%d-%d/%d:%d:%d.%d", &y, &mon, &d, &h, &m, &s, &ms); - - if (n<6 || n>7) - { - gLog << warn << "'" << str << "' no valid Time... ignored." << endl; - return MTime(); - } - - MTime t; - t.Set(y, mon, d, h, m, s, ms); - return t; -} - -void GetTimeFromFile(const char *fname, MTime *start, MTime *stop) -{ - TFile f(fname, "READ"); - - TTree *t = (TTree*)f.Get("RunHeaders"); - if (t->GetEntries()!=1) - { - gLog << warn << "WARNING - File " << fname << " contains no or more than one entry in RunHeaders... Times unchanged." << endl; - return; - } - - MRawRunHeader *h = 0; - t->SetBranchAddress("MRawRunHeader.", &h); - t->GetEntry(0); - if (!h) - { - gLog << warn << "WARNING - File " << fname << " did not contain RunHeaders.MRawRunHeader... Times unchanged." << endl; - return; - } - - if (start && !*start) - *start = h->GetRunStart(); - if (stop && !*stop) - *stop = h->GetRunEnd(); -} - -void Add(MReportFileReadCC *r, const TString &rep, const TString &only) -{ - if (!only.IsNull() && rep!=only) - return; - - r->AddToList(Form("MReport%s", rep.Data())); +static bool HasExtension(const TString &name) +{ + return + name.EndsWith(".rep") || name.EndsWith(".txt") || + name.EndsWith(".raw") || name.EndsWith(".raw.gz"); } @@ -181,7 +151,8 @@ } + const Bool_t kDebugMem = arg.HasOnlyAndRemove("--debug-mem"); const Int_t kComprlvl = arg.GetIntAndRemove("-c", 2); const Bool_t kInterleave = arg.GetIntAndRemove("--interleave=", 1); - const Bool_t kForce = arg.HasOnlyAndRemove("-f"); + const Bool_t kOverwrite = arg.HasOnlyAndRemove("-f"); const Bool_t kForceProc = arg.HasOnlyAndRemove("-ff"); const Int_t run = arg.GetIntAndRemove("--run=", -1); @@ -196,14 +167,12 @@ const Int_t kTelescope = arg.GetIntAndRemove("--telescope=", -1); Bool_t kUpdate = arg.HasOnlyAndRemove("--update") || arg.HasOnlyAndRemove("-u"); - const TString kOnly = arg.GetStringAndRemove("--only=", ""); - - MTime kTimeStart; - MTime kTimeStop; - if (arg.HasOption("--star=")) - kTimeStart = AnalyseTime(arg.GetStringAndRemove("--start=")); - if (arg.HasOption("--stop=")) - kTimeStop = AnalyseTime(arg.GetStringAndRemove("--stop=")); - -// const TString kSqlDataBase(arg.GetStringAndRemove("--sql=")); + const TString kOnly = arg.GetStringAndRemove("--only", ""); + const Bool_t kNullOut = arg.HasOnlyAndRemove("--dev-null"); + +// Int_t kDebugEnv = arg.HasOnlyAndRemove("--debug-env") ? 1 : 0; +// kDebugEnv = arg.GetIntAndRemove("--debug-env=", kDebugEnv); + + MTime kTimeStart(arg.GetStringAndRemove("--start=")); + MTime kTimeStop(arg.GetStringAndRemove("--stop=")); if (arg.HasOnlyAndRemove("--sumfile")) @@ -221,5 +190,5 @@ // check for the right usage of the program // - if (arg.GetNumArguments()<1 || arg.GetNumArguments()>2) + if (arg.GetNumArguments()<1 || arg.GetNumArguments()>3) { Usage(); @@ -230,22 +199,34 @@ // This is to make argv[i] more readable insidethe code // - TString kNamein = arg.GetArgumentStr(0); - TString kNameout = arg.GetArgumentStr(1); - - const Bool_t isreport = kNamein.EndsWith(".rep"); - const Bool_t isdc = kNamein.EndsWith(".txt"); - const Bool_t israw = !isreport && !isdc; - - if (!kNamein.EndsWith(".raw") && !kNamein.EndsWith(".raw.gz") && israw) - kNamein += ".raw.gz"; - - if (kNameout.IsNull()) - kNameout = kNamein(0, kNamein.Last('.')); - - if (!kNameout.EndsWith(".root")) - kNameout += ".root"; - -// if (!kSqlDataBase.IsNull() && !israw) -// gLog << warn << "WARNING - Option '--sql=' only valid for raw-files... ignored." << endl; + const Int_t narg = arg.GetNumArguments(); + + const TString arg0 = arg.GetArgumentStr(0); + const TString arg1 = arg.GetArgumentStr(1); + const TString arg2 = arg.GetArgumentStr(2); + + const Bool_t isseq = + (narg==1 && !(HasExtension(arg0) && !arg0.EndsWith(".txt"))) || + (narg==2 && !HasExtension(arg1)) || + (narg==3); + + TString sequence, kNamein, kNameout; + if (isseq) + { + sequence = arg0; + kNameout = arg1; + kNamein = arg2; + + if (HasExtension(arg2)) // set --summary + kHeaderRun = 0; + + kUpdate = kTRUE; + } + else + { + kNamein = arg0; + kNameout = arg1(0, kNamein.Last('.')); + if (!kNameout.EndsWith(".root")) + kNameout += ".root"; + } // @@ -253,45 +234,4 @@ // gROOT->SetBatch(); - - // - // check whether the given files are OK. - // - if (gSystem->AccessPathName(kNamein, kFileExists)) - { - gLog << err << "Sorry, the input file '" << kNamein << "' doesn't exist." << endl; - return 2; - } - - const Bool_t fileexist = !gSystem->AccessPathName(kNameout, kFileExists); - const Bool_t writeperm = !gSystem->AccessPathName(kNameout, kWritePermission); - - if (fileexist && !writeperm) - { - gLog << err << "Sorry, you don't have write permission for '" << kNameout << "'." << endl; - return 2; - } - - if (fileexist && !kUpdate && !kForce) - { - gLog << err << "Sorry, file '" << kNameout << "' already existing." << endl; - return 2; - } - - if (!fileexist && kUpdate) - { - gLog << warn << "File '" << kNameout << "' doesn't yet exist." << endl; - kUpdate=kFALSE; - } - - // - // Evaluate possible start-/stop-time - // - if ((kAutoTimeStart || kAutoTimeStop) && kUpdate && (isreport || isdc)) - GetTimeFromFile(kNameout, kAutoTimeStart?&kTimeStart:0, kAutoTimeStop?&kTimeStop:0); - - if (kTimeStart) - gLog << inf << "Start Time: " << kTimeStart << endl; - if (kTimeStop) - gLog << inf << "Stop Time: " << kTimeStop << endl; // @@ -301,174 +241,44 @@ MParContainer::Class()->IgnoreTObjectStreamer(); - // - // create a (empty) list of parameters which can be used by the tasks - // and an (empty) list of tasks which should be executed - // - MParList plist; - - MTaskList tasks; - tasks.SetOwner(); - plist.AddToList(&tasks); - - // - // ---- The following is only necessary to supress some output ---- - /* - MRawRunHeader runheader; - plist.AddToList(&runheader); - - MRawEvtHeader evtheader; - plist.AddToList(&evtheader); - - MRawEvtData evtdata; - plist.AddToList(&evtdata); - - MRawCrateArray cratearray; - plist.AddToList(&cratearray); - - MTime evttime; - plist.AddToList(&evttime); - */ - - // - // create the tasks which should be executed and add them to the list - // in the case you don't need parameter containers, all of them can - // be created by MRawFileRead::PreProcess - // - const TString option(kUpdate ? "UPDATE" : "RECREATE"); - - MTask *read = 0; - MFilter *filter = 0; - MWriteRootFile *write = new MWriteRootFile(kNameout, option, "Magic root-file", kComprlvl); - - if (isreport || isdc) - { - if (isdc) + if (kDebugMem) + TObject::SetObjectStat(kTRUE); + + { + MJMerpp merpp(Form("MJMerpp %s", arg0.Data())); + merpp.SetOverwrite(kOverwrite); + merpp.SetCompression(kComprlvl); + merpp.SetUpdate(kUpdate); + merpp.SetInterleave(kInterleave); + merpp.SetForceProcessing(kForceProc); + merpp.SetConstrainHeader(kTelescope, kHeaderRun, kHeaderFile); + merpp.SetConstrainRunRep(kRunNumber, kFileNumber); + merpp.SetOnly(kOnly); + merpp.SetTime(kTimeStart, kTimeStop); + merpp.SetAutoTime(kAutoTimeStart, kAutoTimeStop); + merpp.SetNullOut(kNullOut); + //merpp.SetEnvDebug(kDebugEnv); + //merpp.SetEnv(&env); + //merpp.SetDisplay(d);; + + merpp.SetPathIn(kNamein); + merpp.SetPathOut(kNameout); + + const Int_t rc = sequence.IsNull() ? merpp.Process() : merpp.ProcessSeq(sequence); + if (rc>0) { - write->AddContainer("MTimeCurrents", "Currents"); - write->AddContainer("MCameraDC", "Currents"); - write->AddContainer("MReportCurrents", "Currents"); + gLog << err << "Merpp failed." << endl << endl; + return rc; } - else - { - const Bool_t required = kOnly.IsNull(); - write->AddContainer("MReportCamera", "Camera", required); - write->AddContainer("MTimeCamera", "Camera", required); - write->AddContainer("MCameraAUX", "Camera", required); - write->AddContainer("MCameraCalibration", "Camera", required); - write->AddContainer("MCameraCooling", "Camera", required); - write->AddContainer("MCameraActiveLoad", "Camera", required); - write->AddContainer("MCameraHV", "Camera", required); - write->AddContainer("MCameraDC", "Camera", required); - write->AddContainer("MCameraLV", "Camera", required); - write->AddContainer("MCameraLids", "Camera", required); - write->AddContainer("MReportTrigger", "Trigger", required); - write->AddContainer("MTimeTrigger", "Trigger", required); - write->AddContainer("MTriggerBit", "Trigger", required); - write->AddContainer("MTriggerIPR", "Trigger", required); - write->AddContainer("MTriggerCell", "Trigger", required); - write->AddContainer("MTriggerPrescFact", "Trigger", required); - write->AddContainer("MTriggerLiveTime", "Trigger", required); - write->AddContainer("MReportDrive", "Drive", required); - write->AddContainer("MTimeDrive", "Drive", required); - write->AddContainer("MCameraTH", "Rec", required); - write->AddContainer("MCameraTD", "Rec", required); - write->AddContainer("MCameraRecTemp", "Rec", required); - write->AddContainer("MReportRec", "Rec", required); - write->AddContainer("MTimeRec", "Rec", required); - write->AddContainer("MReportCC", "CC", required); - write->AddContainer("MTimeCC", "CC", required); - write->AddContainer("MReportStarguider", "Starguider", required); - write->AddContainer("MTimeStarguider", "Starguider", required); - write->AddContainer("MReportPyrometer", "Pyrometer", required); - write->AddContainer("MTimePyrometer", "Pyrometer", required); - // write->AddContainer("MReportDAQ", "DAQ"); - // write->AddContainer("MTimeDAQ", "DAQ"); - } - - MReportFileReadCC *r = new MReportFileReadCC(kNamein); - r->SetTimeStart(kTimeStart); - r->SetTimeStop(kTimeStop); - if (isdc) - { - r->SetHasNoHeader(); - r->AddToList("MReportCurrents"); - } - else - { - r->SetTelescope(kTelescope); - r->SetRunNumber(kHeaderRun); - r->SetFileNumber(kHeaderFile); - Add(r, "CC", kOnly); - Add(r, "Rec", kOnly); - Add(r, "Drive", kOnly); - Add(r, "Camera", kOnly); - Add(r, "Trigger", kOnly); - Add(r, "Starguider", kOnly); - Add(r, "Pyrometer", kOnly); - // Add(r, "DAQ", kUpdateOnly); - if (kRunNumber>0) - { - const TString f1 = kRunNumber>0 ? Form("MReportRun.fRunNumber==%d", kRunNumber) : ""; - const TString f2 = kFileNumber>0 ? Form("MReportRun.fFileNumber==%d", kFileNumber) : ""; - - const TString f = Form(kRunNumber>0 && kFileNumber>0 ? "%s && %s" : "%s%s", - f1.Data(), f2.Data()); - - r->AddToList("MReportRun"); - filter = new MFDataPhrase(f.Data()); - write->SetFilter(filter); - } - } - read = r; - } - else - { - read = new MRawFileRead(kNamein); - static_cast(read)->SetInterleave(kInterleave); - static_cast(read)->SetForceMode(kForceProc); - - write->AddContainer("MRawRunHeader", "RunHeader"); - write->AddContainer("MTime", "Events"); - write->AddContainer("MRawEvtHeader", "Events"); - write->AddContainer("MRawEvtData", "Events"); - write->AddContainer("MRawEvtData2", "Events"); - write->AddContainer("MRawCrateArray", "Events"); - } - - tasks.AddToList(read); - if (filter) - tasks.AddToList(filter); - /* - if (israw && !kSqlDataBase.IsNull()) - { - MSqlInsertRun *ins = new MSqlInsertRun(kSqlDataBase); - ins->SetUpdate(); - tasks.AddToList(ins); - }*/ - tasks.AddToList(write); - - // - // create the looping object and tell it about the parameters to use - // and the tasks to execute - // - MEvtLoop magic; - magic.SetParList(&plist); - - // - // Start the eventloop which reads the raw file (MRawFileRead) and - // write all the information into a root file (MRawFileWrite) - // - // between reading and writing we can do, transformations, checks, etc. - // (I'm think of a task like MRawDataCheck) - // - if (!magic.Eventloop()) - { - gLog << err << "ERROR: Merging and preprocessing failed!" << endl; - return 2; - } - - tasks.PrintStatistics(); - - gLog << all << "Merpp finished successfull!" << endl; + + //if (kDebugEnv>0) + // env.PrintUntouched(); + } + + if (TObject::GetObjectStat()) + { + TObject::SetObjectStat(kFALSE); + gObjectTable->Print(); + } + return 0; } Index: /trunk/MagicSoft/Mars/mjobs/JobsLinkDef.h =================================================================== --- /trunk/MagicSoft/Mars/mjobs/JobsLinkDef.h (revision 9017) +++ /trunk/MagicSoft/Mars/mjobs/JobsLinkDef.h (revision 9018) @@ -19,4 +19,5 @@ #pragma link C++ class MJCut+; #pragma link C++ class MJSpectrum+; +#pragma link C++ class MJMerpp+; #endif Index: /trunk/MagicSoft/Mars/mjobs/MJMerpp.cc =================================================================== --- /trunk/MagicSoft/Mars/mjobs/MJMerpp.cc (revision 9018) +++ /trunk/MagicSoft/Mars/mjobs/MJMerpp.cc (revision 9018) @@ -0,0 +1,376 @@ +/* ======================================================================== *\ +! +! * +! * This file is part of MARS, the MAGIC Analysis and Reconstruction +! * Software. It is distributed to you in the hope that it can be a useful +! * and timesaving tool in analysing Data of imaging Cerenkov telescopes. +! * It is distributed WITHOUT ANY WARRANTY. +! * +! * Permission to use, copy, modify and distribute this software and its +! * documentation for any purpose is hereby granted without fee, +! * provided that the above copyright notice appear in all copies and +! * that both that copyright notice and this permission notice appear +! * in supporting documentation. It is provided "as is" without express +! * or implied warranty. +! * +! +! +! Author(s): Thomas Bretz, 7/2008 +! +! Copyright: MAGIC Software Development, 2000-2008 +! +! +\* ======================================================================== */ + +///////////////////////////////////////////////////////////////////////////// +// +// MJMerpp +// +// Merging and preprocessing +// +///////////////////////////////////////////////////////////////////////////// +#include "MJMerpp.h" + +#include +#include + +#include "MLogManip.h" + +#include "MParList.h" +#include "MTaskList.h" +#include "MEvtLoop.h" + +#include "MFDataPhrase.h" + +#include "MRawRunHeader.h" + +#include "MRawFileRead.h" +#include "MReportFileReadCC.h" +#include "MWriteRootFile.h" + +ClassImp(MJMerpp); + +using namespace std; + +MJMerpp::MJMerpp(const char *name, const char *title) +{ + fName = name ? name : "MJMerpp"; + fTitle = title ? title : "Standard program for merging subsystem data"; +} + +void MJMerpp::AddTree(MReportFileReadCC &read, const TString &rep, const TString &only) const +{ + if (!only.IsNull() && rep!=only) + return; + + read.AddToList(Form("MReport%s", rep.Data())); +} + +MFilter *MJMerpp::SetupCurrents(MReportFileReadCC &read, MWriteRootFile &write) const +{ + read.SetHasNoHeader(); + read.AddToList("MReportCurrents"); + + write.AddContainer("MTimeCurrents", "Currents"); + write.AddContainer("MCameraDC", "Currents"); + write.AddContainer("MReportCurrents", "Currents"); + + return 0; +} + +MFilter *MJMerpp::SetupReports(MReportFileReadCC &read, MWriteRootFile &write) const +{ + read.SetTelescope(fTelescope); + + read.SetRunNumber(fHeaderRun); + read.SetFileNumber(fHeaderFile); + + AddTree(read, "CC", fOnly); + AddTree(read, "Rec", fOnly); + AddTree(read, "Drive", fOnly); + AddTree(read, "Camera", fOnly); + AddTree(read, "Trigger", fOnly); + AddTree(read, "Starguider", fOnly); + AddTree(read, "Pyrometer", fOnly); + // AddTree(read, "DAQ", fOnly); + + const Bool_t required = fOnly.IsNull(); + write.AddContainer("MReportCamera", "Camera", required); + write.AddContainer("MTimeCamera", "Camera", required); + write.AddContainer("MCameraAUX", "Camera", required); + write.AddContainer("MCameraCalibration", "Camera", required); + write.AddContainer("MCameraCooling", "Camera", required); + write.AddContainer("MCameraActiveLoad", "Camera", required); + write.AddContainer("MCameraHV", "Camera", required); + write.AddContainer("MCameraDC", "Camera", required); + write.AddContainer("MCameraLV", "Camera", required); + write.AddContainer("MCameraLids", "Camera", required); + write.AddContainer("MReportTrigger", "Trigger", required); + write.AddContainer("MTimeTrigger", "Trigger", required); + write.AddContainer("MTriggerBit", "Trigger", required); + write.AddContainer("MTriggerIPR", "Trigger", required); + write.AddContainer("MTriggerCell", "Trigger", required); + write.AddContainer("MTriggerPrescFact", "Trigger", required); + write.AddContainer("MTriggerLiveTime", "Trigger", required); + write.AddContainer("MReportDrive", "Drive", required); + write.AddContainer("MTimeDrive", "Drive", required); + write.AddContainer("MCameraTH", "Rec", required); + write.AddContainer("MCameraTD", "Rec", required); + write.AddContainer("MCameraRecTemp", "Rec", required); + write.AddContainer("MReportRec", "Rec", required); + write.AddContainer("MTimeRec", "Rec", required); + write.AddContainer("MReportCC", "CC", required); + write.AddContainer("MTimeCC", "CC", required); + write.AddContainer("MReportStarguider", "Starguider", required); + write.AddContainer("MTimeStarguider", "Starguider", required); + write.AddContainer("MReportPyrometer", "Pyrometer", required); + write.AddContainer("MTimePyrometer", "Pyrometer", required); + // write.AddContainer("MReportDAQ", "DAQ"); + // write.AddContainer("MTimeDAQ", "DAQ"); + + if (fReportRun<=0) + return 0; + + const TString f1 = fReportRun>0 ? Form("MReportRun.fRunNumber==%d", fReportRun) : ""; + const TString f2 = fReportFile>0 ? Form("MReportRun.fFileNumber==%d", fReportFile) : ""; + + const TString f = Form(fReportRun>0 && fReportFile>0 ? "%s && %s" : "%s%s", + f1.Data(), f2.Data()); + + read.AddToList("MReportRun"); + + MFilter *filter = new MFDataPhrase(f.Data()); + write.SetFilter(filter); + return filter; +} + +void MJMerpp::SetupRaw(MRawFileRead &read, MWriteRootFile &write) const +{ + read.SetInterleave(fInterleave); + read.SetForceMode(fForceProcessing); + + write.AddContainer("MRawRunHeader", "RunHeader"); + write.AddContainer("MTime", "Events"); + write.AddContainer("MRawEvtHeader", "Events"); + write.AddContainer("MRawEvtData", "Events"); + write.AddContainer("MRawEvtData2", "Events"); + write.AddContainer("MRawCrateArray", "Events"); +} + +Bool_t MJMerpp::GetTimeFromFile() +{ + if (!fAutoStartTime && !fAutoStopTime) + return kTRUE; + + TFile f(fPathOut, "READ"); + if (f.IsZombie()) + { + *fLog << err << "ERROR - File " << fPathOut << " could not be opened." << endl; + return kFALSE; + } + + TTree *t = (TTree*)f.Get("RunHeaders"); + if (t->GetEntries()!=1) + { + *fLog << err << "ERROR - File " << fPathOut << " contains no or more than one entry in RunHeaders... Times unchanged." << endl; + return kFALSE; + } + + MRawRunHeader *h = 0; + t->SetBranchAddress("MRawRunHeader.", &h); + t->GetEntry(0); + if (!h) + { + *fLog << err << "ERROR - File " << fPathOut << " did not contain RunHeaders.MRawRunHeader... Times unchanged." << endl; + return kFALSE; + } + + if (fAutoStartTime) + fTimeStart = h->GetRunStart(); + if (fAutoStopTime) + fTimeStop = h->GetRunEnd(); + + return kTRUE; +} + +Bool_t MJMerpp::CheckFilePermissions() +{ + // + // check whether the given files are OK. + // + if (gSystem->AccessPathName(fPathIn, kFileExists)) + { + *fLog << err << "ERROR - Input file '" << fPathIn << "' not accessible." << endl; + return kFALSE; + } + + const Bool_t fileexist = !gSystem->AccessPathName(fPathOut, kFileExists); + const Bool_t writeperm = !gSystem->AccessPathName(fPathOut, kWritePermission); + + if (fileexist && !writeperm) + { + *fLog << err << "ERROR - No write permission for '" << fPathOut << "'." << endl; + return kFALSE; + } + + if (fileexist && !fUpdate && !fOverwrite) + { + *fLog << err << "ERROR - File '" << fPathOut << "' already existing." << endl; + return kFALSE; + } + + if (!fileexist && fUpdate) + { + *fLog << err << "ERROR - File '" << fPathOut << "' doesn't yet exist... no update possible." << endl; + return kFALSE; + } + + return kTRUE; +} + +Int_t MJMerpp::Process() +{ + *fLog << inf; + fLog->Separator(GetDescriptor()); + *fLog << "In: " << fPathIn << endl;; + *fLog << (fUpdate?"Upd":"Out") << ": " << fPathOut << endl;; + *fLog << endl; + + if (!CheckFilePermissions()) + return 1; + + const Bool_t isreport = fPathIn.EndsWith(".rep"); + const Bool_t isdc = fPathIn.EndsWith(".txt"); + + // + // Evaluate possible start-/stop-time + // + if (fUpdate && (isreport || isdc)) + if (!GetTimeFromFile()) + return 2; + + if (fTimeStart) + *fLog << inf << "Start Time: " << fTimeStart << endl; + if (fTimeStop) + *fLog << inf << "Stop Time: " << fTimeStop << endl; + + // + // create a (empty) list of parameters which can be used by the tasks + // and an (empty) list of tasks which should be executed + // + MParList plist; + + MTaskList tasks; + tasks.SetOwner(); + plist.AddToList(&tasks); + + // + // create the tasks which should be executed and add them to the list + // in the case you don't need parameter containers, all of them can + // be created by MRawFileRead::PreProcess + // + const TString option(fUpdate?"UPDATE":(fOverwrite?"RECREATE":"NEW")); + + MTask *read = 0; + MFilter *filter = 0; + MWriteRootFile *write = new MWriteRootFile(fPathOut, option, "Magic root-file", fCompression); + + if (isreport || isdc) + { + MReportFileReadCC *r = new MReportFileReadCC(fPathIn); + r->SetTimeStart(fTimeStart); + r->SetTimeStop(fTimeStop); + + if (isdc) + filter = SetupCurrents(*r, *write); + else + filter = SetupReports(*r, *write); + + read = r; + } + else + { + MRawFileRead *r = new MRawFileRead(fPathIn); + SetupRaw(*r, *write); + read = r; + } + + tasks.AddToList(read); + if (filter) + tasks.AddToList(filter); + + if (!HasNullOut()) + tasks.AddToList(write); + + // Create and setup the eventloop + MEvtLoop evtloop(fName); + evtloop.SetParList(&plist); + evtloop.SetLogStream(fLog); + //evtloop.SetDisplay(fDisplay); + //if (!SetupEnv(evtloop)) + // return kFALSE; + + // Execute first analysis + if (!evtloop.Eventloop(fMaxEvents)) + { + *fLog << err << GetDescriptor() << ": Failed." << endl; + return 3; + } + + *fLog << all << GetDescriptor() << ": Done." << endl << endl << endl;; + + return 0; +} + +Int_t MJMerpp::ProcessSeq(TString fname) +{ + if (!MSequence::InflateSeq(fname)) + return 4; + + MSequence seq(fname); + if (!seq.IsValid()) + return 5; + + const UInt_t num = seq.GetNumEntries(MSequence::kDat); + + const TString reppath = fPathIn; + const TString calpath = fPathOut; + + TString rc; + for (UInt_t i=0; i