source: trunk/MagicSoft/Mars/mjobs/MSequence.cc@ 8427

Last change on this file since 8427 was 8371, checked in by tbretz, 18 years ago
*** empty log message ***
File size: 18.4 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 8/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19!
20! Copyright: MAGIC Software Development, 2004
21!
22!
23\* ======================================================================== */
24
25/////////////////////////////////////////////////////////////////////////////
26//
27// MSequence
28//
29// This class describes a sequence. For sequences see:
30// http://magic.astro.uni-wuerzburg.de/mars/db/queryseq.html
31//
32// A sequence is a collection of runs which should be used together.
33// Any run can be contained only once.
34//
35// Here is an example how a file describing a sequence could look like:
36//
37// ===========================================================================
38//
39// sequence.txt
40// ------------
41//
42// # Sequence number (identifier)
43// Sequence: 31015
44// # Observation Period (used to get the path-names)
45// Period: 18
46// # Date of sunrise of the observation night
47// Night: 2004-06-24
48//
49// # Start time of the sequence (first data run)
50// Start: 2004-06-24 03:12:42
51// # Run number of last data run in sequence
52// LastRun: 31032
53// # Project name of data-runs of sequence
54// Project: 3EG2033+41
55// # Source name of all runs of sequence
56// Source: 3EG2033+41
57// # Trigger table of data-runs of sequence
58// TriggerTable: L1_4NN:L2_DEFAULT
59// # HV Setting table of data-runs of sequence
60// HvSettings: HVSettings_FF36q
61// # Total number of data-events in sequence
62// NumEvents: 250914
63//
64// # List of all runs of this sequence
65// Runs: 31015 31016 31017 31018 31019 31020 31021 31022 31023 31024 31025 31026 31027 31028 31029 31030 31031 31032
66//
67// # List of all calibration runs of this sequence
68// CalRuns: 31015 31016 31017
69// # List of pedestal runs belonging to the calibration runs of this sequence
70// PedRuns: 31018
71// # List of all data runs belonging to this sequence
72// DatRuns: 31019 31020 31022 31023 31024 31025 31027 31028 31030 31032
73//
74// # List of run types of all runs
75// 31015: C
76// 31016: C
77// 31017: C
78// 31018: P
79// 31019: D
80// 31020: D
81// 31021: P
82// 31022: D
83// 31023: D
84// 31024: D
85// 31025: D
86// 31026: P
87// 31027: D
88// 31028: D
89// 31029: P
90// 31030: D
91// 31031: P
92// 31032: D
93//
94// ===========================================================================
95//
96// For special cases you can also setup a sequence directly from a macro,
97// for example:
98//
99// MDirIter pediter, datiter, caliter;
100//
101// MSequence seq;
102// seq.SetNight("2004-07-06");
103// seq.AddPedRuns(31751);
104// seq.AddCalRuns(31752);
105// seq.AddDatRuns(31753, 31764);
106// seq.SetupPedRuns(pediter);
107// seq.SetupCalRuns(caliter);
108// seq.SetupDatRuns(datiter);
109//
110// or
111//
112// MDirIter iter;
113//
114// MSequence seq;
115// seq.SetNight("2004-07-06");
116// seq.AddRuns(31753, 31764);
117// seq.SetupRuns(iter);
118// seq.SetupPedRuns(iter, "/mypath", "[DPC]");
119//
120/////////////////////////////////////////////////////////////////////////////
121#include "MSequence.h"
122
123#include <stdlib.h>
124
125#include <TEnv.h>
126#include <TRegexp.h>
127#include <TSystem.h> // TSystem::ExpandPath
128
129#include "MLog.h"
130#include "MLogManip.h"
131
132#include "MEnv.h"
133#include "MJob.h"
134#include "MAstro.h"
135#include "MString.h"
136#include "MDirIter.h"
137
138ClassImp(MSequence);
139
140using namespace std;
141
142MSequence::~MSequence()
143{
144 /*
145 TExMapIter iter(&fFileNames);
146
147 Long_t key, val;
148
149 while (iter.Next(key, val))
150 delete (TString*)val;
151 */
152}
153
154
155// --------------------------------------------------------------------------
156//
157// Copy the run numbers from the TString runs into the TArrayI data.
158// Runs which are twice in the list are only added once. In this case
159// a warning is emitted.
160//
161void MSequence::Split(TString &runs, TArrayI &data) const
162{
163 const TRegexp regexp("[0-9]+");
164
165 data.Set(0);
166
167 runs.ReplaceAll("\t", " ");
168 runs = runs.Strip(TString::kBoth);
169
170 while (!runs.IsNull())
171 {
172 const TString num = runs(regexp);
173
174 if (num.IsNull())
175 {
176 *fLog << warn << "WARNING - Run is NaN (not a number): '" << runs << "'" << endl;
177 break;
178 }
179
180 const Int_t run = atoi(num.Data());
181 const Int_t n = data.GetSize();
182
183 // skip already existing entries
184 int i;
185 for (i=0; i<n; i++)
186 if (data[i] == run)
187 break;
188
189 if (i<n)
190 *fLog << warn << "WARNING - Run #" << run << " already in list... skipped." << endl;
191 else
192 {
193 // set new entry
194 data.Set(n+1);
195 data[n] = run;
196 }
197
198 runs.Remove(0, runs.First(num)+num.Length());
199 }
200
201 MJob::SortArray(data);
202}
203
204UInt_t MSequence::SetupRuns(MDirIter &iter, const TArrayI &arr, FileType_t type, const char *path) const
205{
206 TString d(path);
207 if (d.IsNull())
208 d = fDataPath;
209
210 const Bool_t def = d.IsNull();
211
212 // For this particular case we assume that the files are added one by
213 // one without wildcards.
214 const Int_t n0 = iter.GetNumEntries();
215
216 // Setup path
217 if (def)
218 {
219 d = GetStandardPath();
220 switch (type)
221 {
222 case kRawDat:
223 case kRawPed:
224 case kRawCal:
225 case kRawAll:
226 d += "rawfiles/";
227 d += fNight.GetStringFmt("%Y/%m/%d");
228 break;
229 case kRootDat:
230 case kRootPed:
231 case kRootCal:
232 case kRootAll:
233 d += "merpp/";
234 d += fNight.GetStringFmt("%Y/%m/%d");
235 break;
236 case kCalibrated:
237 d += Form("callisto/%04d/%08d", fSequence/10000, fSequence);
238 break;
239 case kImages:
240 d += Form("star/%04d/%08d", fSequence/10000, fSequence);
241 break;
242 }
243 }
244 else
245 gSystem->ExpandPathName(d);
246
247 if (!d.EndsWith("/"))
248 d += '/';
249
250 for (int i=0; i<arr.GetSize(); i++)
251 {
252 // R. DeLosReyes and T. Bretz
253 // Changes to read the DAQ numbering format. Changes takes place
254 // between runs 35487 and 00035488 (2004_08_30)
255 const char *fmt = arr[i]>35487 ? "%08d_%s_*_E" : "%05d_%s_*_E";
256
257 TString n;
258 const char *id="_";
259 switch (type)
260 {
261 case kRawDat:
262 case kRootDat:
263 id = "D";
264 break;
265 case kRawPed:
266 case kRootPed:
267 id = "P";
268 break;
269 case kRawCal:
270 case kRootCal:
271 id = "C";
272 break;
273 case kRawAll:
274 case kRootAll:
275 id = "[PCD]";
276 break;
277 case kCalibrated:
278 id = "Y";
279 break;
280 case kImages:
281 id = "I";
282 break;
283 }
284
285 // Create file name
286 n = fNight.GetStringFmt("%Y%m%d_");
287 n += Form(fmt, arr[i], id);
288
289 switch (type)
290 {
291 case kRawDat:
292 case kRawPed:
293 case kRawCal:
294 case kRawAll:
295 n += ".raw.?g?z?";
296 break;
297 default:
298 n += ".root";
299 }
300
301 // Check existance and accessibility of file
302 MDirIter file(d, n, 0);
303 TString name = file();
304 gSystem->ExpandPathName(name);
305 if (gSystem->AccessPathName(name, kFileExists))
306 {
307 *fLog << err;
308 *fLog << "ERROR - File " << d << n << " not accessible!" << endl;
309 return 0;
310 }
311 if (!file().IsNull())
312 {
313 *fLog << err;
314 *fLog << "ERROR - Searching for file " << d << n << " gave more than one result!" << endl;
315 return 0;
316 }
317
318 // Add Path/File to TIter
319 iter.AddDirectory(d, n, 0);
320 }
321
322 const Int_t n1 = iter.GetNumEntries()-n0;
323 const Int_t n2 = arr.GetSize();
324 if (n1==0)
325 {
326 *fLog << err;
327 *fLog << "ERROR - No input files for sequence #" << GetSequence() << endl;
328 *fLog << " read from " << GetName() << endl;
329 *fLog << " found in" << (def?" default-path ":" ") << d << endl;
330 return 0;
331 }
332
333 if (n1==n2)
334 return n1;
335
336 *fLog << err;
337 *fLog << "ERROR - " << n1 << " input files for sequence #" << GetSequence() << " found in" << endl;
338 *fLog << " " << (def?" default-path ":" ") << d << endl;
339 *fLog << " but " << n2 << " files were defined in sequence file" << endl;
340 *fLog << " " << GetName() << endl;
341 if (fLog->GetDebugLevel()<=4)
342 return 0;
343
344 *fLog << dbg << "Files which are searched for this sequence:" << endl;
345 iter.Print();
346 return 0;
347}
348
349// --------------------------------------------------------------------------
350//
351// Read the file fname as setup file for the sequence.
352//
353//void MSequence::GetFileNames(TEnv &env, const TArrayI &arr)
354//{
355 /*
356 for (int i=0; i<arr.GetSize(); i++)
357 {
358 // Get run number
359 const Int_t num = arr[i];
360
361 // Check if name already set
362 if (fFileNames.GetValue(num))
363 continue;
364
365 TString *str = new TString(env.GetValue(Form("%d", num), ""));
366 fFileNames.Add(num, (Long_t)str);
367 }
368 */
369//}
370
371// --------------------------------------------------------------------------
372//
373// Get a file name corresponding to the run-number num, returns 0 if n/a
374//
375//const char *MSequence::GetFileName(UInt_t num)
376//{
377// return 0;
378 /*
379 TString *str = (TString*)fFileNames.GetValue(num);
380 return str ? str->Data() : 0;*/
381//}
382
383MSequence::LightCondition_t MSequence::ReadLightCondition(TEnv &env) const
384{
385 TString str = env.GetValue("LightConditions", "n/a");
386 if (!str.CompareTo("n/a", TString::kIgnoreCase))
387 return kNA;
388 if (!str.CompareTo("No_Moon", TString::kIgnoreCase))
389 return kNoMoon;
390 if (!str.CompareTo("Twilight", TString::kIgnoreCase))
391 return kTwilight;
392 if (!str.CompareTo("Moon", TString::kIgnoreCase))
393 return kMoon;
394 if (!str.CompareTo("Day", TString::kIgnoreCase))
395 return kDay;
396
397 gLog << warn;
398 gLog << "WARNING - in " << fFileName << ":" << endl;
399 gLog << " LightCondition-tag is '" << str << "' but must be n/a, no_moon, twilight, moon or day." << endl;
400 return kNA;
401}
402
403// --------------------------------------------------------------------------
404//
405// Read the file fname as setup file for the sequence.
406//
407MSequence::MSequence(const char *fname, const char *path)
408{
409 fName = fname;
410 fTitle = path;
411
412 fFileName = fname;
413 fDataPath = path;
414
415 gSystem->ExpandPathName(fName);
416 gSystem->ExpandPathName(fTitle);
417
418 const Bool_t rc1 = gSystem->AccessPathName(fName, kFileExists);
419 const Bool_t rc2 = !fTitle.IsNull() && gSystem->AccessPathName(fTitle, kFileExists);
420
421 if (rc1)
422 gLog << err << "ERROR - Sequence file '" << fName << "' doesn't exist." << endl;
423 if (rc2)
424 gLog << err << "ERROR - Directory '" << fTitle << "' doesn't exist." << endl;
425
426 MEnv env(fName);
427
428 fSequence = env.GetValue("Sequence", -1);
429 if (rc1 || rc2)
430 fSequence = (UInt_t)-1;
431
432 fLastRun = env.GetValue("LastRun", -1);
433 fNumEvents = env.GetValue("NumEvents", -1);
434 fPeriod = env.GetValue("Period", -1);
435
436 fLightCondition = ReadLightCondition(env);
437
438 TString str;
439 str = env.GetValue("Start", "");
440 fStart.SetSqlDateTime(str);
441 str = env.GetValue("Night", "");
442 str += " 00:00:00";
443 fNight.SetSqlDateTime(str);
444
445 fProject = env.GetValue("Project", "");
446 fSource = env.GetValue("Source", "");
447 fTriggerTable = env.GetValue("TriggerTable", "");
448 fHvSettings = env.GetValue("HvSettings", "");
449
450 str = env.GetValue("Runs", "");
451 Split(str, fRuns);
452 str = env.GetValue("CalRuns", "");
453 Split(str, fCalRuns);
454 str = env.GetValue("PedRuns", "");
455 Split(str, fPedRuns);
456 str = env.GetValue("DatRuns", "");
457 Split(str, fDatRuns);
458
459 // GetFileNames(env, fRuns);
460 // GetFileNames(env, fCalRuns);
461 // GetFileNames(env, fPedRuns);
462 // GetFileNames(env, fDatRuns);
463
464 // Dummies:
465 env.GetValue("ZdMin", 0);
466 env.GetValue("ZdMax", 0);
467 env.GetValue("L1TriggerTable", 0);
468 env.GetValue("L2TriggerTable", 0);
469
470 if (env.GetNumUntouched()>0)
471 {
472 gLog << warn << "WARNING - At least one resource in the dataset-file has not been touched!" << endl;
473 env.PrintUntouched();
474 }
475}
476
477// --------------------------------------------------------------------------
478//
479// Print the contents of the sequence
480//
481void MSequence::Print(Option_t *o) const
482{
483 gLog << all;
484 if (!IsValid())
485 {
486 gLog << "Sequence: " << fFileName << " <invalid>" << endl;
487 return;
488 }
489 gLog << "Sequence: " << fSequence << endl;
490 gLog << "Period: " << fPeriod << endl;
491 gLog << "Night: " << fNight << endl << endl;
492 gLog << "LightCondition: ";
493 switch (fLightCondition)
494 {
495 case kNA: gLog << "n/a" << endl; break;
496 case kNoMoon: gLog << "NoMoon" << endl; break;
497 case kTwilight: gLog << "Twilight" << endl; break;
498 case kMoon: gLog << "Moon" << endl; break;
499 case kDay: gLog << "Day" << endl; break;
500 }
501 gLog << "Start: " << fStart << endl;
502 gLog << "LastRun: " << fLastRun << endl;
503 gLog << "NumEvents: " << fNumEvents << endl;
504 gLog << "Project: " << fProject << endl;
505 gLog << "Source: " << fSource << endl;
506 gLog << "TriggerTable: " << fTriggerTable << endl;
507 gLog << "HvSettings: " << fHvSettings << endl << endl;
508 gLog << "Runs:";
509 for (int i=0; i<fRuns.GetSize(); i++)
510 gLog << " " << fRuns[i];
511 gLog << endl;
512 gLog << "CalRuns:";
513 for (int i=0; i<fCalRuns.GetSize(); i++)
514 gLog << " " << fCalRuns[i];
515 gLog << endl;
516 gLog << "PedRuns:";
517 for (int i=0; i<fPedRuns.GetSize(); i++)
518 gLog << " " << fPedRuns[i];
519 gLog << endl;
520 gLog << "DatRuns:";
521 for (int i=0; i<fDatRuns.GetSize(); i++)
522 gLog << " " << fDatRuns[i];
523 gLog << endl;
524
525 if (!fDataPath.IsNull())
526 gLog << endl << "DataPath: " << fDataPath << endl;
527}
528
529// --------------------------------------------------------------------------
530//
531// Add all ped runs from the sequence to MDirIter.
532// If path==0 fDataPath is used instead. If it is also empty
533// the standard path of the data-center is assumed.
534// If you have the runs locally use path="."
535// Using raw=kTRUE you get correspodning raw-files setup.
536// Return the number of files added.
537//
538UInt_t MSequence::SetupPedRuns(MDirIter &iter, const char *path, Bool_t raw) const
539{
540 return SetupRuns(iter, fPedRuns, raw?kRawPed:kRootPed, path);
541}
542
543// --------------------------------------------------------------------------
544//
545// Add all data runs from the sequence to MDirIter.
546// If path==0 fDataPath is used instead. If it is also empty
547// the standard path of the data-center is assumed.
548// If you have the runs locally use path="."
549// Using raw=kTRUE you get correspodning raw-files setup.
550// Return the number of files added.
551//
552UInt_t MSequence::SetupDatRuns(MDirIter &iter, const char *path, Bool_t raw) const
553{
554 return SetupRuns(iter, fDatRuns, raw?kRawDat:kRootDat, path);
555}
556
557// --------------------------------------------------------------------------
558//
559// Add all runs from the sequence to MDirIter.
560// If path==0 fDataPath is used instead. If it is also empty
561// the standard path of the data-center is assumed.
562// If you have the runs locally use path="."
563// Using raw=kTRUE you get correspodning raw-files setup.
564// Return the number of files added.
565//
566UInt_t MSequence::SetupAllRuns(MDirIter &iter, const char *path, Bool_t raw) const
567{
568 return SetupRuns(iter, fRuns, raw?kRawAll:kRootAll, path);
569}
570
571// --------------------------------------------------------------------------
572//
573// Add all calibration runs from the sequence to MDirIter.
574// If path==0 fDataPath is used instead. If it is also empty
575// the standard path of the data-center is assumed.
576// If you have the runs locally use path="."
577// Using raw=kTRUE you get correspodning raw-files setup.
578// Return the number of files added.
579//
580UInt_t MSequence::SetupCalRuns(MDirIter &iter, const char *path, Bool_t raw) const
581{
582 return SetupRuns(iter, fCalRuns, raw?kRawCal:kRootCal, path);
583}
584
585// --------------------------------------------------------------------------
586//
587// Add all data runs from the sequence to MDirIter.
588// If path==0 fDataPath is used instead. If it is also empty
589// the standard path of the data-center is assumed.
590// If you have the runs locally use path="."
591// Using raw=kTRUE you get correspodning raw-files setup.
592// Return the number of files added.
593//
594UInt_t MSequence::SetupDatRuns(MDirIter &iter, FileType_t type, const char *path) const
595{
596 return SetupRuns(iter, fDatRuns, type, path);
597}
598
599// --------------------------------------------------------------------------
600//
601// If you want to add runs manually, use this function.
602//
603UInt_t MSequence::AddRuns(UInt_t first, UInt_t last, TArrayI *runs)
604{
605 if (last<first)
606 {
607 *fLog << warn << "MSequence::AddRuns - WARNING: Last runnumber " << last;
608 *fLog << " smaller than first " << first << "... ignored." << endl;
609 return 0;
610 }
611 if (!IsValid())
612 {
613 *fLog << inf << "Setting Sequence number to #" << first << endl;
614 fSequence = first;
615 }
616
617 const UInt_t nall = fRuns.GetSize();
618 const UInt_t nrun = runs ? runs->GetSize() : 0;
619 const UInt_t add = last-first+1;
620
621 fRuns.Set(nall+add);
622 if (runs)
623 runs->Set(nrun+add);
624
625 for (UInt_t i=0; i<add; i++)
626 {
627 fRuns[nall+i] = first+i;
628 if (runs)
629 (*runs)[nrun+i] = first+i;
630 }
631 return add;
632}
633
634// --------------------------------------------------------------------------
635//
636// If you want to change or set the night manually.
637// The Format is
638// SetNight("yyyy-mm-dd");
639//
640void MSequence::SetNight(const char *txt)
641{
642 TString night(txt);
643 night += " 00:00:00";
644 fNight.SetSqlDateTime(night);
645
646 fPeriod = MAstro::GetMagicPeriod(fNight.GetMjd());
647}
Note: See TracBrowser for help on using the repository browser.