source: trunk/MagicSoft/Mars/mjobs/MSequence.cc@ 8742

Last change on this file since 8742 was 8695, checked in by tbretz, 18 years ago
*** empty log message ***
File size: 19.9 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 8/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19!
20! Copyright: MAGIC Software Development, 2004
21!
22!
23\* ======================================================================== */
24
25/////////////////////////////////////////////////////////////////////////////
26//
27// MSequence
28//
29// This class describes a sequence. For sequences see:
30// http://magic.astro.uni-wuerzburg.de/mars/db/queryseq.html
31//
32// A sequence is a collection of runs which should be used together.
33// Any run can be contained only once.
34//
35// Here is an example how a file describing a sequence could look like:
36//
37// ===========================================================================
38//
39// sequence.txt
40// ------------
41//
42// # Sequence number (identifier)
43// Sequence: 31015
44// # Observation Period (used to get the path-names)
45// Period: 18
46// # Date of sunrise of the observation night
47// Night: 2004-06-24
48//
49// # Start time of the sequence (first data run)
50// Start: 2004-06-24 03:12:42
51// # Run number of last data run in sequence
52// LastRun: 31032
53// # Project name of data-runs of sequence
54// Project: 3EG2033+41
55// # Source name of all runs of sequence
56// Source: 3EG2033+41
57// # Trigger table of data-runs of sequence
58// TriggerTable: L1_4NN:L2_DEFAULT
59// # HV Setting table of data-runs of sequence
60// HvSettings: HVSettings_FF36q
61// # Total number of data-events in sequence
62// NumEvents: 250914
63//
64// # List of all runs of this sequence
65// Runs: 31015 31016 31017 31018 31019 31020 31021 31022 31023 31024 31025 31026 31027 31028 31029 31030 31031 31032
66//
67// # List of all calibration runs of this sequence
68// CalRuns: 31015 31016 31017
69// # List of pedestal runs belonging to the calibration runs of this sequence
70// PedRuns: 31018
71// # List of all data runs belonging to this sequence
72// DatRuns: 31019 31020 31022 31023 31024 31025 31027 31028 31030 31032
73//
74// # List of run types of all runs
75// 31015: C
76// 31016: C
77// 31017: C
78// 31018: P
79// 31019: D
80// 31020: D
81// 31021: P
82// 31022: D
83// 31023: D
84// 31024: D
85// 31025: D
86// 31026: P
87// 31027: D
88// 31028: D
89// 31029: P
90// 31030: D
91// 31031: P
92// 31032: D
93//
94// ===========================================================================
95//
96// For special cases you can also setup a sequence directly from a macro,
97// for example:
98//
99// MDirIter pediter, datiter, caliter;
100//
101// MSequence seq;
102// seq.SetNight("2004-07-06");
103// seq.AddPedRuns(31751);
104// seq.AddCalRuns(31752);
105// seq.AddDatRuns(31753, 31764);
106// seq.SetupPedRuns(pediter);
107// seq.SetupCalRuns(caliter);
108// seq.SetupDatRuns(datiter);
109//
110// or
111//
112// MDirIter iter;
113//
114// MSequence seq;
115// seq.SetNight("2004-07-06");
116// seq.AddRuns(31753, 31764);
117// seq.SetupRuns(iter);
118// seq.SetupPedRuns(iter, "/mypath", "[DPC]");
119//
120// ===========================================================================
121//
122// Class Version 2:
123// + fMonteCarlo
124//
125// Class Version 3:
126// + fComment
127//
128/////////////////////////////////////////////////////////////////////////////
129#include "MSequence.h"
130
131#include <stdlib.h>
132
133#include <TEnv.h>
134#include <TRegexp.h>
135#include <TSystem.h> // TSystem::ExpandPath
136
137#include "MLog.h"
138#include "MLogManip.h"
139
140#include "MEnv.h"
141#include "MJob.h"
142#include "MAstro.h"
143#include "MString.h"
144#include "MDirIter.h"
145
146ClassImp(MSequence);
147
148using namespace std;
149
150MSequence::~MSequence()
151{
152 /*
153 TExMapIter iter(&fFileNames);
154
155 Long_t key, val;
156
157 while (iter.Next(key, val))
158 delete (TString*)val;
159 */
160}
161
162
163// --------------------------------------------------------------------------
164//
165// Copy the run numbers from the TString runs into the TArrayI data.
166// Runs which are twice in the list are only added once. In this case
167// a warning is emitted.
168//
169void MSequence::Split(TString &runs, TArrayI &data) const
170{
171 const TRegexp regexp("[0-9]+");
172
173 data.Set(0);
174
175 runs.ReplaceAll("\t", " ");
176 runs = runs.Strip(TString::kBoth);
177
178 while (!runs.IsNull())
179 {
180 const TString num = runs(regexp);
181
182 if (num.IsNull())
183 {
184 *fLog << warn << "WARNING - Run is NaN (not a number): '" << runs << "'" << endl;
185 break;
186 }
187
188 const Int_t run = atoi(num.Data());
189 const Int_t n = data.GetSize();
190
191 // skip already existing entries
192 int i;
193 for (i=0; i<n; i++)
194 if (data[i] == run)
195 break;
196
197 if (i<n)
198 *fLog << warn << "WARNING - Run #" << run << " already in list... skipped." << endl;
199 else
200 {
201 // set new entry
202 data.Set(n+1);
203 data[n] = run;
204 }
205
206 runs.Remove(0, runs.First(num)+num.Length());
207 }
208
209 MJob::SortArray(data);
210}
211
212UInt_t MSequence::SetupRuns(MDirIter &iter, const TArrayI &arr, FileType_t type, const char *path) const
213{
214 TString d(path);
215 if (d.IsNull())
216 d = fDataPath;
217
218 const Bool_t def = d.IsNull();
219
220 // For this particular case we assume that the files are added one by
221 // one without wildcards.
222 const Int_t n0 = iter.GetNumEntries();
223
224 // Setup path
225 if (def)
226 {
227 d = GetStandardPath();
228 switch (type)
229 {
230 case kRawDat: // rawdata
231 case kRawPed:
232 case kRawCal:
233 case kRawAll:
234 case kRootDat: // mcdata
235 case kRootPed:
236 case kRootCal:
237 case kRootAll:
238 d += "rawfiles/";
239 d += fNight.GetStringFmt("%Y/%m/%d");
240 break;
241 case kCalibrated:
242 d += Form("callisto/%04d/%08d", fSequence/10000, fSequence);
243 break;
244 case kImages:
245 d += Form("star/%04d/%08d", fSequence/10000, fSequence);
246 break;
247 }
248 }
249 else
250 gSystem->ExpandPathName(d);
251
252 if (!d.EndsWith("/"))
253 d += '/';
254
255 for (int i=0; i<arr.GetSize(); i++)
256 {
257 // R. DeLosReyes and T. Bretz
258 // Changes to read the DAQ numbering format. Changes takes place
259 // between runs 35487 and 00035488 (2004_08_30)
260 const char *fmt = arr[i]>35487 || fMonteCarlo ? "%08d_%s_*_E" : "%05d_%s_*_E";
261
262 TString n;
263 const char *id="_";
264 switch (type)
265 {
266 case kRawDat:
267 case kRootDat:
268 id = "D";
269 break;
270 case kRawPed:
271 case kRootPed:
272 id = "P";
273 break;
274 case kRawCal:
275 case kRootCal:
276 id = "C";
277 break;
278 case kRawAll:
279 case kRootAll:
280 id = "[PCD]";
281 break;
282 case kCalibrated:
283 id = "Y";
284 break;
285 case kImages:
286 id = "I";
287 break;
288 }
289
290 // Create file name
291 n = fNight.GetStringFmt("%Y%m%d_");
292 n += Form(fmt, arr[i], id);
293
294 switch (type)
295 {
296 case kRawDat:
297 case kRawPed:
298 case kRawCal:
299 case kRawAll:
300 n += ".raw.?g?z?";
301 break;
302 default:
303 n += ".root";
304 }
305
306 // Check existance and accessibility of file
307 MDirIter file(d, n, 0);
308 TString name = file();
309 gSystem->ExpandPathName(name);
310 if (gSystem->AccessPathName(name, kFileExists))
311 {
312 *fLog << err;
313 *fLog << "ERROR - File " << d << n << " not accessible!" << endl;
314 return 0;
315 }
316 if (!file().IsNull())
317 {
318 *fLog << err;
319 *fLog << "ERROR - Searching for file " << d << n << " gave more than one result!" << endl;
320 return 0;
321 }
322
323 // Add Path/File to TIter
324 iter.AddDirectory(d, n, 0);
325 }
326
327 const Int_t n1 = iter.GetNumEntries()-n0;
328 const Int_t n2 = arr.GetSize();
329 if (n1==0)
330 {
331 *fLog << err;
332 *fLog << "ERROR - No input files for sequence #" << GetSequence() << endl;
333 *fLog << " read from " << GetName() << endl;
334 *fLog << " found in" << (def?" default-path ":" ") << d << endl;
335 return 0;
336 }
337
338 if (n1==n2)
339 return n1;
340
341 *fLog << err;
342 *fLog << "ERROR - " << n1 << " input files for sequence #" << GetSequence() << " found in" << endl;
343 *fLog << " " << (def?" default-path ":" ") << d << endl;
344 *fLog << " but " << n2 << " files were defined in sequence file" << endl;
345 *fLog << " " << GetName() << endl;
346 if (fLog->GetDebugLevel()<=4)
347 return 0;
348
349 *fLog << dbg << "Files which are searched for this sequence:" << endl;
350 iter.Print();
351 return 0;
352}
353
354// --------------------------------------------------------------------------
355//
356// Read the file fname as setup file for the sequence.
357//
358//void MSequence::GetFileNames(TEnv &env, const TArrayI &arr)
359//{
360 /*
361 for (int i=0; i<arr.GetSize(); i++)
362 {
363 // Get run number
364 const Int_t num = arr[i];
365
366 // Check if name already set
367 if (fFileNames.GetValue(num))
368 continue;
369
370 TString *str = new TString(env.GetValue(Form("%d", num), ""));
371 fFileNames.Add(num, (Long_t)str);
372 }
373 */
374//}
375
376// --------------------------------------------------------------------------
377//
378// Get a file name corresponding to the run-number num, returns 0 if n/a
379//
380//const char *MSequence::GetFileName(UInt_t num)
381//{
382// return 0;
383 /*
384 TString *str = (TString*)fFileNames.GetValue(num);
385 return str ? str->Data() : 0;*/
386//}
387
388MSequence::LightCondition_t MSequence::ReadLightCondition(TEnv &env) const
389{
390 TString str = env.GetValue("LightConditions", "n/a");
391 if (!str.CompareTo("n/a", TString::kIgnoreCase))
392 return kNA;
393 if (!str.CompareTo("No_Moon", TString::kIgnoreCase))
394 return kNoMoon;
395 if (!str.CompareTo("Twilight", TString::kIgnoreCase))
396 return kTwilight;
397 if (!str.CompareTo("Moon", TString::kIgnoreCase))
398 return kMoon;
399 if (!str.CompareTo("Day", TString::kIgnoreCase))
400 return kDay;
401
402 gLog << warn;
403 gLog << "WARNING - in " << fFileName << ":" << endl;
404 gLog << " LightCondition-tag is '" << str << "' but must be n/a, no_moon, twilight, moon or day." << endl;
405 return kNA;
406}
407
408// --------------------------------------------------------------------------
409//
410// Read the file fname as setup file for the sequence.
411//
412MSequence::MSequence(const char *fname, const char *path)
413{
414 fName = fname;
415 fTitle = path;
416
417 fFileName = fname;
418 fDataPath = path;
419
420 gSystem->ExpandPathName(fName);
421 gSystem->ExpandPathName(fTitle);
422
423 const Bool_t rc1 = gSystem->AccessPathName(fName, kFileExists);
424 const Bool_t rc2 = !fTitle.IsNull() && gSystem->AccessPathName(fTitle, kFileExists);
425
426 if (rc1)
427 gLog << err << "ERROR - Sequence file '" << fName << "' doesn't exist." << endl;
428 if (rc2)
429 gLog << err << "ERROR - Directory '" << fTitle << "' doesn't exist." << endl;
430
431 MEnv env(fName);
432
433 fSequence = env.GetValue("Sequence", -1);
434 if (rc1 || rc2)
435 fSequence = (UInt_t)-1;
436
437 fLastRun = env.GetValue("LastRun", -1);
438 fNumEvents = env.GetValue("NumEvents", -1);
439 fPeriod = env.GetValue("Period", -1);
440
441 fLightCondition = ReadLightCondition(env);
442
443 TString str;
444 str = env.GetValue("Start", "");
445 fStart.SetSqlDateTime(str);
446 str = env.GetValue("Night", "");
447 str += " 00:00:00";
448 fNight.SetSqlDateTime(str);
449
450 fProject = env.GetValue("Project", "");
451 fSource = env.GetValue("Source", "");
452 fTriggerTable = env.GetValue("TriggerTable", "");
453 fHvSettings = env.GetValue("HvSettings", "");
454 fMonteCarlo = env.GetValue("MonteCarlo", kFALSE);
455 fComment = env.GetValue("Comment", "");
456
457 str = env.GetValue("Runs", "");
458 Split(str, fRuns);
459 str = env.GetValue("CalRuns", "");
460 Split(str, fCalRuns);
461 str = env.GetValue("PedRuns", "");
462 Split(str, fPedRuns);
463 str = env.GetValue("DatRuns", "");
464 Split(str, fDatRuns);
465
466 // GetFileNames(env, fRuns);
467 // GetFileNames(env, fCalRuns);
468 // GetFileNames(env, fPedRuns);
469 // GetFileNames(env, fDatRuns);
470
471 // Dummies:
472 env.GetValue("ZdMin", 0);
473 env.GetValue("ZdMax", 0);
474 env.GetValue("L1TriggerTable", 0);
475 env.GetValue("L2TriggerTable", 0);
476
477 if (env.GetNumUntouched()>0)
478 {
479 gLog << warn << "WARNING - At least one resource in the dataset-file has not been touched!" << endl;
480 env.PrintUntouched();
481 }
482}
483
484//---------------------------------------------------------------------------
485//
486// Make sure that the name used for writing doesn't contain a full path
487//
488const char *MSequence::GetName() const
489{
490 const char *pos = strrchr(GetRcName(), '/');
491 return pos>0 ? pos+1 : GetRcName();
492}
493
494
495// --------------------------------------------------------------------------
496//
497// Print the contents of the sequence
498//
499void MSequence::Print(Option_t *o) const
500{
501 gLog << all;
502 if (!IsValid())
503 {
504 gLog << "Sequence: " << fFileName << " <invalid>" << endl;
505 return;
506 }
507 gLog << "# Path: " << GetRcName() << endl;
508 gLog << "# Name: " << GetName() << endl;
509 gLog << endl;
510 gLog << "Sequence: " << fSequence << endl;
511 if (fMonteCarlo)
512 gLog << "MonteCarlo: Yes" << endl;
513 gLog << "Period: " << fPeriod << endl;
514 gLog << "Night: " << fNight << endl << endl;
515 gLog << "LightCondition: ";
516 switch (fLightCondition)
517 {
518 case kNA: gLog << "n/a" << endl; break;
519 case kNoMoon: gLog << "NoMoon" << endl; break;
520 case kTwilight: gLog << "Twilight" << endl; break;
521 case kMoon: gLog << "Moon" << endl; break;
522 case kDay: gLog << "Day" << endl; break;
523 }
524 gLog << "Start: " << fStart << endl;
525 gLog << "LastRun: " << fLastRun << endl;
526 gLog << "NumEvents: " << fNumEvents << endl;
527 gLog << "Project: " << fProject << endl;
528 gLog << "Source: " << fSource << endl;
529 gLog << "TriggerTable: " << fTriggerTable << endl;
530 gLog << "HvSettings: " << fHvSettings << endl << endl;
531 gLog << "Runs:";
532 for (int i=0; i<fRuns.GetSize(); i++)
533 gLog << " " << fRuns[i];
534 gLog << endl;
535 gLog << "CalRuns:";
536 for (int i=0; i<fCalRuns.GetSize(); i++)
537 gLog << " " << fCalRuns[i];
538 gLog << endl;
539 gLog << "PedRuns:";
540 for (int i=0; i<fPedRuns.GetSize(); i++)
541 gLog << " " << fPedRuns[i];
542 gLog << endl;
543 gLog << "DatRuns:";
544 for (int i=0; i<fDatRuns.GetSize(); i++)
545 gLog << " " << fDatRuns[i];
546 gLog << endl;
547
548 if (!fDataPath.IsNull())
549 gLog << endl << "DataPath: " << fDataPath << endl;
550
551 gLog << endl << "Comment: " << fComment << endl;
552
553}
554
555// --------------------------------------------------------------------------
556//
557// Add all ped runs from the sequence to MDirIter.
558// If path==0 fDataPath is used instead. If it is also empty
559// the standard path of the data-center is assumed.
560// If you have the runs locally use path="."
561// Using raw=kTRUE you get correspodning raw-files setup.
562// Return the number of files added.
563//
564UInt_t MSequence::SetupPedRuns(MDirIter &iter, const char *path, Bool_t raw) const
565{
566 return SetupRuns(iter, fPedRuns, raw?kRawPed:kRootPed, path);
567}
568
569// --------------------------------------------------------------------------
570//
571// Add all data runs from the sequence to MDirIter.
572// If path==0 fDataPath is used instead. If it is also empty
573// the standard path of the data-center is assumed.
574// If you have the runs locally use path="."
575// Using raw=kTRUE you get correspodning raw-files setup.
576// Return the number of files added.
577//
578UInt_t MSequence::SetupDatRuns(MDirIter &iter, const char *path, Bool_t raw) const
579{
580 return SetupRuns(iter, fDatRuns, raw?kRawDat:kRootDat, path);
581}
582
583// --------------------------------------------------------------------------
584//
585// Add all runs from the sequence to MDirIter.
586// If path==0 fDataPath is used instead. If it is also empty
587// the standard path of the data-center is assumed.
588// If you have the runs locally use path="."
589// Using raw=kTRUE you get correspodning raw-files setup.
590// Return the number of files added.
591//
592UInt_t MSequence::SetupAllRuns(MDirIter &iter, const char *path, Bool_t raw) const
593{
594 return SetupRuns(iter, fRuns, raw?kRawAll:kRootAll, path);
595}
596
597// --------------------------------------------------------------------------
598//
599// Add all calibration runs from the sequence to MDirIter.
600// If path==0 fDataPath is used instead. If it is also empty
601// the standard path of the data-center is assumed.
602// If you have the runs locally use path="."
603// Using raw=kTRUE you get correspodning raw-files setup.
604// Return the number of files added.
605//
606UInt_t MSequence::SetupCalRuns(MDirIter &iter, const char *path, Bool_t raw) const
607{
608 return SetupRuns(iter, fCalRuns, raw?kRawCal:kRootCal, path);
609}
610
611// --------------------------------------------------------------------------
612//
613// Add all data runs from the sequence to MDirIter.
614// If path==0 fDataPath is used instead. If it is also empty
615// the standard path of the data-center is assumed.
616// If you have the runs locally use path="."
617// Using raw=kTRUE you get correspodning raw-files setup.
618// Return the number of files added.
619//
620UInt_t MSequence::SetupDatRuns(MDirIter &iter, FileType_t type, const char *path) const
621{
622 return SetupRuns(iter, fDatRuns, type, path);
623}
624
625// --------------------------------------------------------------------------
626//
627// If you want to add runs manually, use this function.
628//
629UInt_t MSequence::AddRuns(UInt_t first, UInt_t last, TArrayI *runs)
630{
631 if (last<first)
632 {
633 *fLog << warn << "MSequence::AddRuns - WARNING: Last runnumber " << last;
634 *fLog << " smaller than first " << first << "... ignored." << endl;
635 return 0;
636 }
637 if (!IsValid())
638 {
639 *fLog << inf << "Setting Sequence number to #" << first << endl;
640 fSequence = first;
641 }
642
643 const UInt_t nall = fRuns.GetSize();
644 const UInt_t nrun = runs ? runs->GetSize() : 0;
645 const UInt_t add = last-first+1;
646
647 fRuns.Set(nall+add);
648 if (runs)
649 runs->Set(nrun+add);
650
651 for (UInt_t i=0; i<add; i++)
652 {
653 fRuns[nall+i] = first+i;
654 if (runs)
655 (*runs)[nrun+i] = first+i;
656 }
657 return add;
658}
659
660// --------------------------------------------------------------------------
661//
662// If you want to change or set the night manually.
663// The Format is
664// SetNight("yyyy-mm-dd");
665//
666void MSequence::SetNight(const char *txt)
667{
668 TString night(txt);
669 night += " 00:00:00";
670 fNight.SetSqlDateTime(night);
671
672 fPeriod = MAstro::GetMagicPeriod(fNight.GetMjd());
673}
674
675// --------------------------------------------------------------------------
676//
677// If the sequence name seq is just a digit it is inflated to a full
678// path following the datacenter standard.
679//
680// Returns if file accessible or not.
681//
682Bool_t MSequence::InflatePath(TString &seq, Bool_t ismc)
683{
684 if (seq.IsDigit())
685 {
686 const Int_t numseq = seq.Atoi();
687 seq = "/magic/";
688 if (ismc)
689 seq += "montecarlo/";
690 seq += Form("sequences/%04d/sequence%08d.txt", numseq/10000, numseq);
691 gLog << inf << "Inflated sequence file: " << seq << endl;
692 }
693
694 if (!gSystem->AccessPathName(seq, kFileExists))
695 return kTRUE;
696
697 gLog << err << "Sorry, sequence file '" << seq << "' doesn't exist." << endl;
698 return kFALSE;
699}
Note: See TracBrowser for help on using the repository browser.