source: trunk/MagicSoft/Mars/datacenter/macros/filldotrun.C@ 9336

Last change on this file since 9336 was 9297, checked in by hoehne, 17 years ago
*** empty log message ***
File size: 23.4 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
19! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// filldotrun.C
29// ============
30//
31// This macro is used in the datacenter to automatically fill the run-database
32// with the information stored in the .run-files written by the central
33// control.
34//
35// To following Arehucas versions are Currently supported:
36// 040505-0, 040514-0,
37// 040518-0, 040727-0,
38// 041113-0, 041209-0, 041221-0
39// 050224-0, 050317-0, 050322-0, 050401-0, 050413-0, 050415-0, 050714-0,
40// 050719-0, 050829-0, 051025-0,
41// 060330-0, 060401-0, 060808-0
42// 070416-0,
43// 080220-0, 080519-0, 080912-0, 081204-0, 081214-0,
44// 090203-0
45//
46// Usage:
47// .x filldotrun.C+("/data/MAGIC/Period019/ccdata", kTRUE)
48//
49// While the first argument is the directory in which all subdirectories where
50// searches for CC_*.run files. All these files were analysed and the run
51// info will be put into the DB, eg:
52// "/magic/subsystemdata/cc" would do it for all data
53// "/magic/subsystemdata/cc/2005" for one year
54// "/magic/subsystemdata/cc/2005/11" for one month
55// "/magic/subsystemdata/cc/2005/11/11" for a single day
56// "/magic/subsystemdata/cc/2005/11/11/file.run" for a single file
57//
58// The second argument is the 'dummy-mode'. If it is kTRUE dummy-mode is
59// switched on and nothing will be written into the database. Instead
60// informations about the subtables are displayed. This is usefull for tests
61// when adding a new arehucas version support. If it is kFALSE the information
62// are written into the subtables and the runs info is written into the
63// rundatabase.
64//
65// In the automatic case it makes sense to check the logfiles to make sure
66// that everything is fine...
67//
68// Make sure, that database and password are corretly set in a resource
69// file called sql.rc and the resource file is found.
70//
71// Remark: Running it from the commandline looks like this:
72// root -q -l -b filldotrun.C+\(\"path\"\,kFALSE\) 2>&1 | tee filldotrun.log
73//
74// Returns 0 in case of failure and 1 in case of success.
75//
76/////////////////////////////////////////////////////////////////////////////
77#include <iostream>
78#include <iomanip>
79#include <fstream>
80
81#include <TMath.h>
82#include <TRegexp.h>
83
84#include "MTime.h"
85#include "MDirIter.h"
86#include "MSQLMagic.h"
87
88using namespace std;
89
90Int_t insert(MSQLMagic &serv, Bool_t dummy, TString filename)
91{
92 ifstream fin(filename);
93 if (!fin)
94 {
95 cout << "Could not open file " << filename << endl;
96 return -1;
97 }
98 //read the file a second time for the check for number of columns
99 ifstream fin2(filename);
100
101 TString strng;
102 TString strng2;
103 TObjArray *array = new TObjArray();
104 Int_t check=0;
105
106 strng.ReadLine(fin);
107 strng2.ReadLine(fin2);
108 if (strng!=TString("[CC Plain Run Summary File]"))
109 {
110 cout << filename << ": No Plain Run Summary File" << endl;
111 cout << "First Line: " << strng << endl;
112 cout << endl;
113 return -1;
114 }
115
116 strng.ReadLine(fin);
117 strng2.ReadLine(fin2);
118 TRegexp reg("[0-9][0-9][0-9][0-9][0-9][0-9]-[0-9]");
119 TString arehucas = strng(reg);
120 arehucas.Prepend("20");
121 arehucas.ReplaceAll("-", "");
122
123 Int_t version = atoi(arehucas.Data());
124 if (version!=200405050 && version!=200405140 && version!=200405180 &&
125 version!=200407270 && version!=200411130 && version!=200412090 &&
126 version!=200412210 &&
127 version!=200502240 && version!=200503170 && version!=200503220 &&
128 version!=200504010 && version!=200504130 && version!=200504150 &&
129 version!=200507140 && version!=200507190 && version!=200508290 &&
130 version!=200510250 &&
131 version!=200603300 && version!=200604010 && version!=200608080 &&
132 version!=200704160 &&
133 version!=200802200 && version!=200805190 && version!=200809120 &&
134 version!=200812040 && version!=200812140 &&
135 version!=200902030)
136 {
137 cout << filename << ": File Version unknown - please update the macro!" << endl;
138 cout << "Second Line: " << strng << endl;
139 cout << endl;
140 return -1;
141 }
142
143 if (version >= 200805190)
144 {
145 strng.ReadLine(fin);
146 strng2.ReadLine(fin2);
147 if (!strng.BeginsWith("Telescope M"))
148 {
149 cout << "WARNING - Line 3 doesn't start with 'Telescope M'." << endl;
150 cout << strng << endl;
151 }
152 }
153
154 if (version >= 200411130)
155 {
156 strng.ReadLine(fin);
157 strng2.ReadLine(fin2);
158 if (strng[0]!='#')
159 {
160 cout << "WARNING - '#' expected." << endl;
161 cout << strng << endl;
162 }
163 }
164
165 cout << " * V" << version << " " << endl;
166
167 Int_t cnt=0;
168 while (1)
169 {
170 // ===== Check for number of columns in file =====
171 // different arehucas versions provide a different number of columns:
172 // 200405050 - 200405140: 18 columns
173 // 200405180 - 200407270: 35 columns
174 // 200411130 - 200510250: 43 columns
175 // 200603300 - 200802200: 52 columns
176 // 200805190 - 200809120: 54 columns
177 // > 200812040: 55 columns
178 //
179
180 strng2.ReadLine(fin2);
181 //fill an array with the values separated by ' '
182 array=strng2.Tokenize(" ");
183 check=array->GetEntries();
184 cout << check << endl;
185 //check the number of columns for the different versions, if there is an empty line (check=0), do nothing
186 if (check != 0)
187 {
188 if ((version <= 200405140 && check != 18) ||
189 (version > 200405140 && version <= 200407270 && check != 35) ||
190 (version > 200407270 && version <= 200510250 && check != 43) ||
191 (version > 200510250 && version <= 200802200 && check != 52) ||
192 (version > 200802200 && version <= 200809120 && check != 54) ||
193 (version > 200809120 && check != 55))
194 {
195 strng.ReadLine(fin);
196 cout << "ERROR - Number of columns doesn't match number of required columns." << endl;
197 array->Delete();
198 continue;
199 }
200 }
201 array->Delete();
202
203
204 // ========== Col 1: Telescope Number =========
205 Int_t telnumber = 1; // FIXME: "NULL"?
206 if (version >=200805190)
207 {
208 strng.ReadToDelim(fin, ' ');
209 if (!fin)
210 break;
211 if (strng[0]!='M')
212 {
213 cout << "WARNING - First character is not an M." << endl;
214 cout << strng << endl;
215 strng.ReadLine(fin);
216 continue;
217 }
218 if (strng[1]!='1')
219 {
220 cout << "WARNING - Only MAGIC 1 implemented so far." << endl;
221 cout << strng << endl;
222 strng.ReadLine(fin);
223 continue;
224 }
225
226 telnumber = atoi(strng.Data()+1);
227 }
228
229 // ========== Col 2: Run Number =========
230 //Reading the line
231 //and converting some strings to ints/floats
232 strng.ReadToDelim(fin, ' ');
233 if (!fin)
234 break;
235
236 Int_t runnumber = atoi(strng.Data());
237
238 //runnumber=0 means no valid dataset
239 //-> continue
240 if (runnumber == 0)
241 {
242 strng.ReadLine(fin);
243 cout << "WARNING - Runnumber == 0" << endl;
244 cout << strng << endl;
245 continue;
246 }
247
248 // ========== Col 3: Subrun Number ========= starting with version 200805190
249 Int_t filenumber = 0; // FIXME: "NULL"?
250 if (version >=200805190)
251 {
252 strng.ReadToDelim(fin, ' ');
253 filenumber = atoi(strng.Data());
254 }
255
256 TString where = Form("fTelescopeNumber=%d AND fFileNumber=%d",
257 telnumber, filenumber);
258 if (serv.ExistStr("fRunNumber", "RunData", Form("%d", runnumber), where))
259 {
260 // FIXME: Maybe we can implement a switch to update mode?
261 cout << "WARNING - Entry M" << telnumber << ":" << runnumber << "/" << filenumber << " already existing... skipped." << endl;
262 strng.ReadLine(fin);
263 continue;
264 }
265
266 // ========== Col 4: Run Type =========
267 strng.ReadToDelim(fin, ' ');
268 if (strng.Contains("???"))
269 strng="n/a";
270
271 Int_t runtype = serv.QueryKeyOfName("RunType", strng, kFALSE);
272 if (runtype<0)
273 {
274 cout << "ERROR - RunType " << strng << " not available." << endl;
275 strng.ReadLine(fin);
276 continue;
277 }
278
279 //cout << runtype << " ";
280
281 // ========== Col 5,6: Start Time =========
282 TString startdate, starttime;
283 startdate.ReadToDelim(fin, ' ');
284 starttime.ReadToDelim(fin, ' ');
285 //cout << startdate << " " << starttime << " ";
286
287 // ========== Col 7,8: Stop Time =========
288 TString stopdate, stoptime;
289 stopdate.ReadToDelim(fin, ' ');
290 stoptime.ReadToDelim(fin, ' ');
291 //cout << stopdate << " " << stoptime << " ";
292
293 if (startdate.Contains("???"))
294 startdate="0000-00-00";
295 if (starttime.Contains("???"))
296 starttime="00:00:00";
297 if (stopdate.Contains("???"))
298 stopdate="0000-00-00";
299 if (stoptime.Contains("???"))
300 stoptime="00:00:00";
301
302 // ========== Col 9: Source Name =========
303 strng.ReadToDelim(fin, ' ');
304 if (strng.Contains("???"))
305 strng="Unavailable";
306
307 Int_t sourcekey = serv.QueryKeyOfName("Source", strng.Data());
308 if (sourcekey<0)
309 {
310 strng.ReadLine(fin);
311 continue;
312 }
313 //cout << sourcekey << " ";
314
315 // ========== Col 10,11: Local source position =========
316 strng.ReadToDelim(fin, ' ');
317 Float_t zd = atof(strng.Data());
318
319 strng.ReadToDelim(fin, ' ');
320 Float_t az = atof(strng.Data());
321
322 //cout << zd << " " << az << " ";
323
324 // ========== Col 12: Number of Events =========
325 strng.ReadToDelim(fin, ' ');
326 Int_t evtno = atoi(strng.Data());
327
328 //cout << evtno << " ";
329
330 // ========== Col 13: Project Name =========
331 strng.ReadToDelim(fin, ' ');
332 if (strng.Contains("???"))
333 strng="Unavailable";
334
335 Int_t projkey = serv.QueryKeyOfName("Project", strng);
336 if (projkey<0)
337 {
338 strng.ReadLine(fin);
339 continue;
340 }
341 //cout << projkey << " ";
342
343 // ========== Col 14: Trigger Table Name =========
344 // starting from version 200411130: Col 14,15: Trigger Table Name =========
345 strng.ReadToDelim(fin, ' ');
346 if (strng.Contains("???"))
347 strng="n/a";
348
349 Int_t l1triggerkey=1;
350 Int_t l2triggerkey=1;
351 if (version >=200411130)
352 {
353 l1triggerkey = serv.QueryKeyOfName("L1TriggerTable", strng);
354 if (l1triggerkey<0)
355 {
356 strng.ReadLine(fin);
357 continue;
358 }
359
360 strng.ReadToDelim(fin, ' ');
361 if (strng.Contains("???"))
362 strng="n/a";
363
364 l2triggerkey = serv.QueryKeyOfName("L2TriggerTable", strng);
365 if (l2triggerkey<0)
366 {
367 strng.ReadLine(fin);
368 continue;
369 }
370 }
371 else
372 {
373 Int_t c=0;
374
375 if (strng.Contains(":"))
376 c=1;
377
378 if (strng.Contains("L1_") && !(strng.Contains(":")))
379 c=2;
380
381 if (strng.Contains("n/a"))
382 c=3;
383
384 switch (c)
385 {
386 case 0:
387 {
388 l2triggerkey = serv.QueryKeyOfName("L2TriggerTable", strng);
389 if (l2triggerkey<0)
390 {
391 strng.ReadLine(fin);
392 continue;
393 }
394
395 strng="n/a";
396 l1triggerkey = 1;
397
398 break;
399 }
400 case 1:
401 {
402 TString L1TT, L2TT;
403 L2TT=strng(7,12);
404 L1TT=strng(0,6);
405
406 l1triggerkey = serv.QueryKeyOfName("L1TriggerTable", L1TT);
407 if (l1triggerkey<0)
408 {
409 strng.ReadLine(fin);
410 continue;
411 }
412
413 l2triggerkey = serv.QueryKeyOfName("L2TriggerTable", L2TT);
414 if (l2triggerkey<0)
415 {
416 strng.ReadLine(fin);
417 continue;
418 }
419
420 break;
421 }
422 case 2:
423 {
424 l1triggerkey = serv.QueryKeyOfName("L1TriggerTable", strng);
425 if (l1triggerkey<0)
426 {
427 strng.ReadLine(fin);
428 continue;
429 }
430
431 strng="n/a";
432 l2triggerkey = 1;
433
434 break;
435 }
436 case 3:
437 {
438 l1triggerkey = 1;
439 l2triggerkey = 1;
440 break;
441 }
442 default:
443 {
444 cout << "WARNING: neither L1 nor L2 Trigger table - please check what is happening." << strng << endl;
445 break;
446 }
447 }
448 }
449
450 // ========== Col 16-18: TrigRate, L2 UnPresc Rate, L2 Presc Rate ==========
451 strng.ReadToDelim(fin, ' ');
452 Float_t trigrate = atof(strng.Data());
453
454 strng.ReadToDelim(fin, ' ');
455 Float_t l2uprate = atof(strng.Data());
456
457 strng.ReadToDelim(fin, ' ');
458 Float_t l2prrate = atof(strng.Data());
459
460 // ========== Col 19,20: DaqRate, Storage Rate ==========
461 strng.ReadToDelim(fin, ' ');
462 Float_t daqrate = atof(strng.Data());
463
464 strng.ReadToDelim(fin, ' ');
465 Float_t storerate = atof(strng.Data());
466
467 // ========== Col 21: HV table =========
468 if (version==200405050 || version==200405140)
469 strng.ReadToDelim(fin, '\n');
470 else
471 strng.ReadToDelim(fin, ' ');
472 if (strng.Contains("???"))
473 strng="n/a";
474
475 Int_t hvkey = serv.QueryKeyOfName("HvSettings", strng);
476 if (hvkey<0)
477 {
478 //strng.ReadLine(fin);
479 continue;
480 }
481
482 if (version==200405180 || version==200407270)
483 strng.ReadLine(fin);
484
485 Int_t testflagkey=1;
486 Int_t lightcondkey=1;
487 Int_t dttablekey=1;
488 Int_t triggerdelaytablekey=1;
489 Int_t calibrationscriptkey=1;
490 if (version>=200411130)
491 {
492 // ========== Col 22-38: DC and HV-values, mjd =========
493 for (int i=0 ; i<17 ; i++)
494 {
495 strng.ReadToDelim(fin, ' ');
496 }
497
498 // ========== Col 39: test-flag =========
499 strng.ReadToDelim(fin, ' ');
500 if (strng.Contains("???"))
501 strng="n/a";
502
503 testflagkey = serv.QueryKeyOfName("TestFlag", strng);
504 if (testflagkey<0)
505 {
506 strng.ReadLine(fin);
507 continue;
508 }
509
510 // ========== Col 40: light conditions =========
511 strng.ReadToDelim(fin, ' ');
512 if (strng.Contains("???"))
513 strng="n/a";
514
515 lightcondkey = serv.QueryKeyOfName("LightConditions", strng);
516 if (lightcondkey<0)
517 {
518 strng.ReadLine(fin);
519 continue;
520 }
521
522 // ========== Col 41: discriminator threshold table =========
523 strng.ReadToDelim(fin, ' ');
524 if (strng.Contains("???"))
525 strng="n/a";
526
527 dttablekey = serv.QueryKeyOfName("DiscriminatorThresholdTable", strng);
528 if (dttablekey<0)
529 {
530 strng.ReadLine(fin);
531 continue;
532 }
533
534 // ========== Col 42: trigger delay table =========
535 strng.ReadToDelim(fin, ' ');
536 if (strng.Contains("???"))
537 strng="n/a";
538
539 triggerdelaytablekey = serv.QueryKeyOfName("TriggerDelayTable", strng);
540 if (triggerdelaytablekey<0)
541 {
542 strng.ReadLine(fin);
543 continue;
544 }
545
546 // ========== Col 43,44: Telescope RA and Dec sent to drive =========
547 strng.ReadToDelim(fin, ' ');
548 strng.ReadToDelim(fin, ' ');
549
550 // ========== Col 45: Calibration Script =========
551 if (version>=200411130 && version<=200510250)
552 strng.ReadToDelim(fin, '\n');
553 else
554 strng.ReadToDelim(fin, ' ');
555 if (strng.Contains("???"))
556 strng="n/a";
557
558 calibrationscriptkey = serv.QueryKeyOfName("CalibrationScript", strng);
559 if (calibrationscriptkey<0)
560 {
561 strng.ReadLine(fin);
562 continue;
563 }
564
565 }
566
567 Int_t observationmodekey=1;
568 if (version>=200603300)
569 {
570 // ========== Col 46: Observation Mode =========
571 strng.ReadToDelim(fin, ' ');
572 if (strng.Contains("???"))
573 strng="n/a";
574
575 observationmodekey = serv.QueryKeyOfName("ObservationMode", strng);
576 if (observationmodekey<0)
577 {
578 strng.ReadLine(fin);
579 continue;
580 }
581
582 // ========== Col 47-54: Source RA and Dec, DT's and IPR =========
583 for (int i=0 ; i<7 ; i++)
584 {
585 strng.ReadToDelim(fin, ' ');
586 }
587 if (version<=200809120)
588 strng.ReadToDelim(fin, '\n');
589 else
590 strng.ReadToDelim(fin, ' ');
591 }
592
593 Int_t sumtriggerflagkey=1;
594 if (version>=200812040)
595 {
596 // ========= Col 55: SumTrigger flag =========
597 strng.ReadToDelim(fin, '\n');
598 if (strng.Contains("???"))
599 strng="n/a";
600
601 sumtriggerflagkey = serv.QueryKeyOfName("SumTriggerFlag", strng);
602 if (sumtriggerflagkey<0)
603 {
604 strng.ReadLine(fin);
605 continue;
606 }
607
608 }
609
610
611 // ================================================================
612 // ========== Data read from file now access the database =========
613 // ================================================================
614
615 //assemble the query that is needed to insert the values of this run
616 TString query;
617 query += Form("fTelescopeNumber=%d, ", telnumber);
618 query += Form("fRunNumber=%d, ", runnumber);
619 query += Form("fFileNumber=%d, ", filenumber);
620 query += Form("fRunTypeKEY=%d, ", runtype);
621 query += Form("fProjectKEY=%d, ", projkey);
622 query += Form("fSourceKEY=%d, ", sourcekey);
623 query += Form("fNumEvents=%d, ", evtno);
624 query += Form("fRunStart=\"%s %s\", ", startdate.Data(), starttime.Data());
625 query += Form("fRunStop=\"%s %s\", ", stopdate.Data(), stoptime.Data());
626 query += Form("fL1TriggerTableKEY=%d, ", l1triggerkey);
627 query += Form("fL2TriggerTableKEY=%d, ", l2triggerkey);
628 query += Form("fTestFlagKEY=%d, ", testflagkey);
629 query += Form("fCalibrationScriptKEY=%d, ", calibrationscriptkey);
630 query += Form("fTriggerDelayTableKEY=%d, ", triggerdelaytablekey);
631 query += Form("fDiscriminatorThresholdTableKEY=%d, ", dttablekey);
632 query += Form("fLightConditionsKEY=%d, ", lightcondkey);
633 query += Form("fHvSettingsKEY=%d, ", hvkey);
634 query += Form("fObservationModeKEY=%d, ", observationmodekey);
635 query += Form("fSumTriggerFlagKEY=%d, ", sumtriggerflagkey);
636 if (!TMath::IsNaN(zd) && TMath::Finite(zd))
637 query += Form("fZenithDistance=%d, ", TMath::Nint(zd));
638 if (!TMath::IsNaN(az) && TMath::Finite(az))
639 query += Form("fAzimuth=%d, ", TMath::Nint(az));
640 if (!TMath::IsNaN(storerate) && TMath::Finite(storerate))
641 query += Form("fDaqStoreRate=%d, ", TMath::Nint(storerate));
642 if (!TMath::IsNaN(daqrate) && TMath::Finite(daqrate))
643 query += Form("fDaqTriggerRate=%d, ", TMath::Nint(daqrate));
644 if (!TMath::IsNaN(trigrate) && TMath::Finite(trigrate))
645 query += Form("fMeanTriggerRate=%d, ", TMath::Nint(trigrate));
646 if (!TMath::IsNaN(l2prrate) && TMath::Finite(l2prrate))
647 query += Form("fL2RatePresc=%d, ", TMath::Nint(l2prrate));
648 if (!TMath::IsNaN(l2uprate) && TMath::Finite(l2uprate))
649 query += Form("fL2RateUnpresc=%d, ", TMath::Nint(l2uprate));
650 query += "fMagicNumberKEY=1, fExcludedFDAKEY=1";
651
652 cnt++;
653
654 //send query, add dataset to DB
655 if (serv.Insert("RunData", query)==kFALSE)
656 return -1;
657
658 TString query2=Form("fTelescopeNumber=%d, fRunNumber=%d, fFileNumber=%d, "
659 "fPriority=%d, fTimingCorrection='1970-01-01 00:00:00', fCompmux='1970-01-01 00:00:00'",
660 telnumber, runnumber, filenumber, runnumber);
661 if (testflagkey==3)
662 query2+=" , fDataCheckDone='1970-01-01 00:00:00'";
663
664 //create entry in table RunProcessStatus for this runnumber
665 if (serv.Insert("RunProcessStatus", query2)==kFALSE)
666 return -1;
667 }
668
669 return cnt;
670}
671
672// This tool will work from Period017 (2004_05_17) on...
673int filldotrun(const TString path="/home/lapalma/transfer/ccdata", Bool_t dummy=kTRUE)
674{
675 MSQLMagic serv("sql.rc");
676 if (!serv.IsConnected())
677 {
678 cout << "ERROR - Connection to database failed." << endl;
679 return 0;
680 }
681
682 cout << "filldotrun" << endl;
683 cout << "----------" << endl;
684 cout << endl;
685 cout << "Connected to " << serv.GetName() << endl;
686 cout << "Search Path: " << path << endl;
687 cout << endl;
688
689 serv.SetIsDummy(dummy);
690
691 if (path.EndsWith(".run"))
692 {
693 cout << path(TRegexp("CC_.*.run", kFALSE)) << flush;
694 Int_t n = insert(serv, dummy, path);
695 cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
696
697 return n<0 ? 2 : 1;
698 }
699
700 MDirIter Next(path, "CC_*.run", -1);
701 while (1)
702 {
703 TString name = Next();
704 if (name.IsNull())
705 break;
706
707 cout << " * " << name(TRegexp("CC_.*.run", kFALSE)) << endl;
708 Int_t n = insert(serv, dummy, name);
709 cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
710
711 if (n<0)
712 return 2;
713 }
714
715 return 1;
716}
Note: See TracBrowser for help on using the repository browser.