source: trunk/MagicSoft/Mars/mjobs/MDataSet.cc@ 7445

Last change on this file since 7445 was 7413, checked in by tbretz, 19 years ago
*** empty log message ***
File size: 15.1 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 1/2005 <mailto:tbretz@astro.uni-wuerzburg.de>
19!
20! Copyright: MAGIC Software Development, 2004-2005
21!
22!
23\* ======================================================================== */
24
25/////////////////////////////////////////////////////////////////////////////
26//
27// MDataSet
28//
29// This class describes a collection of sequences.
30//
31// Such an input file looks like:
32//
33// crab.seq:
34// ---------
35// AnalysisNumber: 1
36//
37// SequencesOn: 35222
38// SequencesOff: 36817
39//
40// Sequence00035222.File: sequences/sequence035222.txt
41// Sequence00036817.File: sequences/sequence036817.txt
42//
43// Sequence00035222.Dir: /data2/wuerzburg/Crab-Analyse/images/035222
44// Sequence00036817.Dir: /data2/wuerzburg/Crab-Analyse/images/036817
45//
46// The analysis number is an artifical number used to name the output
47// files automatically if the names are not overwritten in the corresponding
48// programs.
49//
50// The sequence number are used to concatenate the filenames of the
51// sequences using the file structure used in the datacenter. Each sequence
52// can be added to the on and off data at the same time but only once.
53//
54// If you have different file names you can overwrite the default file names
55// using Sequence%08d.File (make sure you have 8 digits!)
56//
57// In standard coditions (datacenter file system) paths are concatenated
58// by using the information in the sequence files (date, etc). You can
59// overwrite the directories in which the sequence-files (eg I-files) are
60// stored using Sequence%08d.Dir (make sure you have 8 digits!)
61//
62// Resource file entries are case sensitive!
63//
64// IMPORTANT:
65// * Run filenames must begin with a string which allows correct
66// ordering in time, otherwise synchronization might fail.
67// * Sequence filenames should also have names allowing to order them
68// in time, but it is not necessary.
69//
70// MISSING (27/01/04): The default name and paths cannot be used yet, because
71// they have to be defined soon.
72//
73/////////////////////////////////////////////////////////////////////////////
74#include "MDataSet.h"
75
76#include <string.h> // necessary for Fedora core 2 with kernel 2.6.9-1.667 #1 and gcc 3.4.2
77#include <errno.h> // necessary for Fedora core 2 with kernel 2.6.9-1.667 #1 and gcc 3.4.2
78
79#include <stdlib.h>
80#include <fstream>
81
82#include <TEnv.h>
83#include <TChain.h>
84#include <TRegexp.h>
85#include <TSystem.h> // TSystem::ExpandPath
86
87#include "MLog.h"
88#include "MLogManip.h"
89
90#include "MRead.h"
91#include "MJob.h"
92#include "MAstro.h"
93#include "MDirIter.h"
94#include "MSequence.h"
95#include "MPointingPos.h"
96
97ClassImp(MDataSet);
98
99using namespace std;
100
101const TString MDataSet::fgCatalog = "/magic/datacenter/setup/magic_favorites.edb";
102const TString MDataSet::fgPathDataFiles = "/magic/data/star";
103const TString MDataSet::fgPathSequences = "/magic/sequences";
104
105// --------------------------------------------------------------------------
106//
107// Copy the sequence numbers from the TString runs into the TArrayI data
108// Sequences which are twice in the list are only added once. In this case
109// a warning is emitted.
110//
111void MDataSet::Split(TString &runs, TArrayI &data) const
112{
113 const TRegexp regexp("[0-9]+");
114
115 data.Set(0);
116 runs = runs.Strip(TString::kTrailing);
117
118 while (!runs.IsNull())
119 {
120 const TString num = runs(regexp);
121
122 if (num.IsNull())
123 {
124 *fLog << warn << "WARNING - Sequence is NaN (not a number): " << runs << endl;
125 break;
126 }
127
128 const Int_t seq = atoi(num.Data());
129 const Int_t n = data.GetSize();
130
131 // skip already existing entries
132 int i;
133 for (i=0; i<n; i++)
134 if (data[i] == seq)
135 break;
136
137 if (i<n)
138 *fLog << warn << "WARNING - Sequence #" << seq << " already in list... skipped." << endl;
139 else
140 {
141 // set new entry
142 data.Set(n+1);
143 data[n] = seq;
144 }
145
146 // remove entry from string
147 runs.Remove(0, runs.First(num)+num.Length());
148 }
149
150 MJob::SortArray(data);
151}
152
153// --------------------------------------------------------------------------
154//
155// After resolving the sequence filename and directory either from the
156// default (/magic/data/sequences/0004/sequence00004000.txt) or from
157// the corresponding entries in the dataset file.
158// The entries are sorted by filename.
159//
160void MDataSet::ResolveSequences(TEnv &env, const TArrayI &num, TList &list, const TString &sequences, const TString &data) const
161{
162 for (int i=0; i<num.GetSize(); i++)
163 {
164 TString name = env.GetValue(Form("Sequence%08d.File", num[i]), "");
165 TString dir = env.GetValue(Form("Sequence%08d.Dir", num[i]), "");
166
167 // Set default sequence file and dir name
168 if (name.IsNull())
169 name = Form("%s%04d/sequence%08d.txt", sequences.Data(), num[i]/10000, num[i]);
170 if (dir.IsNull())
171 dir = Form("%s%04d/%08d", data.Data(), num[i]/10000, num[i]);
172
173 gSystem->ExpandPathName(name);
174 gSystem->ExpandPathName(dir);
175
176 if (gSystem->AccessPathName(name, kFileExists))
177 gLog << warn << "WARNING - Sequence file '" << name << "' doesn't exist." << endl;
178
179 if (gSystem->AccessPathName(dir, kFileExists))
180 gLog << warn << "WARNING - Directory '" << dir << "' doesn't exist." << endl;
181
182 list.Add(new TNamed(name, dir));
183 }
184
185 // For the synchronization we must make sure, that all sequences are
186 // in the correct order...
187 // list.Sort();
188}
189
190// --------------------------------------------------------------------------
191//
192// Read the file fname as setup file for the sequence.
193//
194MDataSet::MDataSet(const char *fname, TString sequences, TString data)
195{
196 fName = fname;
197
198 fSequencesOn.SetOwner();
199 fSequencesOff.SetOwner();
200
201 const char *expname = gSystem->ExpandPathName(fname);
202
203 fTitle = Form("Dataset contained in file %s", expname);
204
205 const Bool_t access = !gSystem->AccessPathName(expname, kFileExists);
206 if (!access)
207 gLog << err << "ERROR - Dataset file " << expname << " not accessible!" << endl;
208
209 TEnv env(expname);
210 delete [] expname;
211
212 fNumAnalysis = env.GetValue("AnalysisNumber", -1);
213
214 TString str;
215 str = env.GetValue("SequencesOn", "");
216 Split(str, fNumSequencesOn);
217 str = env.GetValue("SequencesOff", "");
218 Split(str, fNumSequencesOff);
219
220 SetupDefaultPath(sequences, fgPathSequences);
221 SetupDefaultPath(data, fgPathDataFiles);
222
223 ResolveSequences(env, fNumSequencesOn, fSequencesOn, sequences, data);
224 ResolveSequences(env, fNumSequencesOff, fSequencesOff, sequences, data);
225
226 fNameSource = env.GetValue("SourceName", "");
227 fCatalog = env.GetValue("Catalog", fgCatalog);
228 fIsWobbleMode = env.GetValue("WobbleMode", kFALSE);
229 fComment = env.GetValue("Comment", "");
230
231 fNameSource = fNameSource.Strip(TString::kBoth);
232 fCatalog = fCatalog.Strip(TString::kBoth);
233}
234
235// --------------------------------------------------------------------------
236//
237// Return '+' if both can be accessed, '-' otherwise.
238//
239void MDataSet::PrintFile(const TObject &obj)
240{
241 const Char_t access =
242 !gSystem->AccessPathName(obj.GetName(), kFileExists) &&
243 !gSystem->AccessPathName(obj.GetTitle(), kFileExists) ? '+' : '-';
244
245 gLog << " " << access << " " << obj.GetName() << " <" << obj.GetTitle() << ">" << endl;
246}
247
248// --------------------------------------------------------------------------
249//
250// Print the contents of the sequence
251//
252void MDataSet::Print(Option_t *o) const
253{
254 gLog << all;
255 if (!IsValid())
256 {
257 gLog << "Dataset: " << fName << " <invalid - no analysis number available>" << endl;
258 return;
259 }
260 gLog << "AnalysisNumber: " << fNumAnalysis << endl << endl;
261 gLog << "SequencesOn: ";
262 for (int i=0; i<fNumSequencesOn.GetSize(); i++)
263 gLog << " " << fNumSequencesOn[i];
264 gLog << endl;
265 gLog << "SequencesOff: ";
266 for (int i=0; i<fNumSequencesOff.GetSize(); i++)
267 gLog << " " << fNumSequencesOff[i];
268 gLog << endl << endl;
269
270 gLog << "SourceName: " << fNameSource << endl;
271 gLog << "Catalog: " << fCatalog << endl;
272
273 gLog << "WobbleMode: " << (fIsWobbleMode?"On":"Off") << endl << endl;
274
275 gLog << "Comment: " << fComment << endl;
276
277 if (TString(o).Contains("files", TString::kIgnoreCase))
278 {
279 TObject *obj=0;
280
281 gLog << endl;
282 gLog << "On-Data Files:" << endl;
283 TIter NextOn(&fSequencesOn);
284 while ((obj=NextOn()))
285 PrintFile(*obj);
286
287 gLog << endl;
288 gLog << "Off-Data Files:" << endl;
289 TIter NextOff(&fSequencesOff);
290 while ((obj=NextOff()))
291 PrintFile(*obj);
292
293 return;
294 }
295}
296
297// --------------------------------------------------------------------------
298//
299// Adds all sequences contained in list to the MDirIter. After adding
300// everything MDirIter::Sort is called to sort all entries by name.
301//
302Bool_t MDataSet::AddSequencesFromList(const TList &list, MDirIter &files)
303{
304 TIter Next(const_cast<TList*>(&list));
305 TObject *o=0;
306 while ((o=Next()))
307 {
308 MSequence seq(o->GetName());
309 if (!seq.IsValid())
310 {
311 gLog << warn << "WARNING - Sequence " << o->GetName() << " invalid!" << endl;
312 return kFALSE;
313 }
314
315 const TString dir(o->GetTitle());
316 seq.SetupDatRuns(files, MSequence::kImages, dir.IsNull() ? 0 : dir.Data());
317 }
318
319 // This is important in case of synchronisation, because the
320 // files in the sequences can be interleaved (eg W1, W2)
321 // Filenames MUST begin with an appropriate string which allow
322 // to order them correctly in time!
323 // files.Sort();
324
325 if (gLog.GetDebugLevel()>4)
326 {
327 gLog << dbg << "Files which are searched:" << endl;
328 files.Print();
329 }
330 return kTRUE;
331}
332
333Bool_t MDataSet::AddFilesOn(MRead &read) const
334{
335 MDirIter files;
336 if (!AddSequencesFromList(fSequencesOn, files))
337 return kFALSE;
338 return read.AddFiles(files)>0;
339}
340
341Bool_t MDataSet::AddFilesOff(MRead &read) const
342{
343 MDirIter files;
344 if (!AddSequencesFromList(fSequencesOff, files))
345 return kFALSE;
346 return read.AddFiles(files)>0;
347}
348
349Bool_t MDataSet::AddFiles(MRead &read) const
350{
351 const Bool_t rc1 = AddFilesOff(read);
352 const Bool_t rc2 = AddFilesOn(read);
353 return rc1 && rc2;
354}
355
356Int_t MDataSet::AddFilesToChain(MDirIter &files, TChain &chain)
357{
358 Int_t num=0;
359 while (1)
360 {
361 const TString fname = files.Next();
362 if (fname.IsNull())
363 break;
364
365 const Int_t n = chain.Add(fname);
366 if (n<=0)
367 return kFALSE;
368 num += n;
369 }
370 return num;
371}
372
373Bool_t MDataSet::AddFilesOn(TChain &chain) const
374{
375 MDirIter files;
376 if (!AddSequencesFromList(fSequencesOn, files))
377 return kFALSE;
378 return AddFilesToChain(files, chain)>0;
379}
380
381Bool_t MDataSet::AddFilesOff(TChain &chain) const
382{
383 MDirIter files;
384 if (!AddSequencesFromList(fSequencesOff, files))
385 return kFALSE;
386 return AddFilesToChain(files, chain)>0;
387}
388
389Bool_t MDataSet::AddFiles(TChain &read) const
390{
391 const Bool_t rc1 = AddFilesOff(read);
392 const Bool_t rc2 = AddFilesOn(read);
393 return rc1 && rc2;
394}
395
396Bool_t MDataSet::GetSourcePos(MPointingPos &pos) const
397{
398 if (!HasSource())
399 {
400 gLog << err << "ERROR - MDataSet::GetSourcePos called, but no source available." << endl;
401 return kFALSE;
402 }
403
404 TString catalog(fCatalog);
405 gSystem->ExpandPathName(catalog);
406
407 ifstream fin(catalog);
408 if (!fin)
409 {
410 gLog << err << "Cannot open file " << catalog << ": ";
411 gLog << strerror(errno) << endl;
412 return kFALSE;
413 }
414
415 TString ra, dec, epoch;
416
417 Int_t n = 0;
418 while (1)
419 {
420 TString line;
421 line.ReadLine(fin);
422 if (!fin)
423 break;
424
425 n++;
426
427 // Strip all spaces from line
428 for (int i=0; i<line.Length(); i++)
429 if (line[i]==' ')
430 line.Remove(i--, 1);
431
432 const Ssiz_t s = line.First(',');
433 if (s<0 || fNameSource!=line(0, s))
434 continue;
435
436 // CrabNebula,f|L|K0,5:34:32.0,22:0:52,-1.0,2000
437
438 for (int i=0; i<6; i++)
439 {
440 const Ssiz_t p = line.First(',');
441 if (p<0 && i<5)
442 {
443 gLog << err << "ERROR - Not enough arguments in line #" << n << " of " << catalog << endl;
444 return kFALSE;;
445 }
446
447 switch (i)
448 {
449 case 0:
450 case 1:
451 case 4:
452 break;
453 case 2:
454 ra = line(0, p);
455 break;
456 case 3:
457 dec = line(0, p);
458 break;
459 case 5:
460 epoch = line;
461 break;
462 }
463 line.Remove(0, p+1);
464 }
465
466 if (line.First(',')>=0)
467 {
468 gLog << err << "ERROR - Too much arguments in line #" << n << " of " << catalog << endl;
469 return kFALSE;
470 }
471
472 break;
473 }
474
475 if (epoch.IsNull())
476 {
477 gLog << err << "ERROR - No entry " << fNameSource << " in " << catalog << endl;
478 return kFALSE;
479 }
480
481 if (epoch!=(TString)"2000")
482 {
483 gLog << err << "ERROR - Epoch not 2000... not supported." << endl;
484 return kFALSE;
485 }
486
487 Double_t r,d;
488 if (!MAstro::Coordinate2Angle(ra, r))
489 {
490 gLog << err << "ERROR - Interpreting right ascension: " << ra << endl;
491 return kFALSE;
492 }
493 if (!MAstro::Coordinate2Angle(dec, d))
494 {
495 gLog << err << "ERROR - Interpreting declination: " << dec << endl;
496 return kFALSE;
497 }
498
499 pos.SetSkyPosition(r, d);
500 pos.SetTitle(fNameSource);
501
502 return kTRUE;
503}
504
505// --------------------------------------------------------------------------
506//
507// Calls ReplaceAll(old, news) for all Dir-entries
508//
509void MDataSet::ReplaceDir(TList &list, const TString &old, const TString &news) const
510{
511 TIter Next(&list);
512 TNamed *name = 0;
513 while ((name=(TNamed*)Next()))
514 {
515 TString dir = name->GetTitle();
516 dir.ReplaceAll(old, news);
517 name->SetTitle(dir);
518 }
519}
520
521// --------------------------------------------------------------------------
522//
523// Calls ReplaceAll(old, news) for all File-entries
524//
525void MDataSet::ReplaceFile(TList &list, const TString &old, const TString &news) const
526{
527 TIter Next(&list);
528 TNamed *name = 0;
529 while ((name=(TNamed*)Next()))
530 {
531 TString file = name->GetName();
532 file.ReplaceAll(old, news);
533 name->SetName(file);
534 }
535}
Note: See TracBrowser for help on using the repository browser.