source: trunk/MagicSoft/Mars/datacenter/macros/filldotrun.C@ 9225

Last change on this file since 9225 was 9213, checked in by hoehne, 16 years ago
*** empty log message ***
File size: 23.4 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
19! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// filldotrun.C
29// ============
30//
31// This macro is used in the datacenter to automatically fill the run-database
32// with the information stored in the .run-files written by the central
33// control.
34//
35// To following Arehucas versions are Currently supported:
36// 040505-0, 040514-0,
37// 040518-0, 040727-0,
38// 041113-0, 041209-0, 041221-0
39// 050224-0, 050317-0, 050322-0, 050401-0, 050413-0, 050415-0, 050714-0,
40// 050719-0, 050829-0, 051025-0,
41// 060330-0, 060401-0, 060808-0
42// 070416-0,
43// 080220-0, 080519-0, 080912-0, 081204-0, 081214-0
44//
45// Usage:
46// .x filldotrun.C+("/data/MAGIC/Period019/ccdata", kTRUE)
47//
48// While the first argument is the directory in which all subdirectories where
49// searches for CC_*.run files. All these files were analysed and the run
50// info will be put into the DB, eg:
51// "/magic/subsystemdata/cc" would do it for all data
52// "/magic/subsystemdata/cc/2005" for one year
53// "/magic/subsystemdata/cc/2005/11" for one month
54// "/magic/subsystemdata/cc/2005/11/11" for a single day
55// "/magic/subsystemdata/cc/2005/11/11/file.run" for a single file
56//
57// The second argument is the 'dummy-mode'. If it is kTRUE dummy-mode is
58// switched on and nothing will be written into the database. Instead
59// informations about the subtables are displayed. This is usefull for tests
60// when adding a new arehucas version support. If it is kFALSE the information
61// are written into the subtables and the runs info is written into the
62// rundatabase.
63//
64// In the automatic case it makes sense to check the logfiles to make sure
65// that everything is fine...
66//
67// Make sure, that database and password are corretly set in a resource
68// file called sql.rc and the resource file is found.
69//
70// Remark: Running it from the commandline looks like this:
71// root -q -l -b filldotrun.C+\(\"path\"\,kFALSE\) 2>&1 | tee filldotrun.log
72//
73// Returns 0 in case of failure and 1 in case of success.
74//
75/////////////////////////////////////////////////////////////////////////////
76#include <iostream>
77#include <iomanip>
78#include <fstream>
79
80#include <TMath.h>
81#include <TRegexp.h>
82
83#include "MTime.h"
84#include "MDirIter.h"
85#include "MSQLMagic.h"
86
87using namespace std;
88
89Int_t insert(MSQLMagic &serv, Bool_t dummy, TString filename)
90{
91 ifstream fin(filename);
92 if (!fin)
93 {
94 cout << "Could not open file " << filename << endl;
95 return -1;
96 }
97 //read the file a second time for the check for number of columns
98 ifstream fin2(filename);
99
100 TString strng;
101 TString strng2;
102 TObjArray *array = new TObjArray();
103 Int_t check=0;
104
105 strng.ReadLine(fin);
106 strng2.ReadLine(fin2);
107 if (strng!=TString("[CC Plain Run Summary File]"))
108 {
109 cout << filename << ": No Plain Run Summary File" << endl;
110 cout << "First Line: " << strng << endl;
111 cout << endl;
112 return -1;
113 }
114
115 strng.ReadLine(fin);
116 strng2.ReadLine(fin2);
117 TRegexp reg("[0-9][0-9][0-9][0-9][0-9][0-9]-[0-9]");
118 TString arehucas = strng(reg);
119 arehucas.Prepend("20");
120 arehucas.ReplaceAll("-", "");
121
122 Int_t version = atoi(arehucas.Data());
123 if (version!=200405050 && version!=200405140 && version!=200405180 &&
124 version!=200407270 && version!=200411130 && version!=200412090 &&
125 version!=200412210 &&
126 version!=200502240 && version!=200503170 && version!=200503220 &&
127 version!=200504010 && version!=200504130 && version!=200504150 &&
128 version!=200507140 && version!=200507190 && version!=200508290 &&
129 version!=200510250 &&
130 version!=200603300 && version!=200604010 && version!=200608080 &&
131 version!=200704160 &&
132 version!=200802200 && version!=200805190 && version!=200809120 &&
133 version!=200812040 && version!=200812140)
134 {
135 cout << filename << ": File Version unknown - please update the macro!" << endl;
136 cout << "Second Line: " << strng << endl;
137 cout << endl;
138 return -1;
139 }
140
141 if (version >= 200805190)
142 {
143 strng.ReadLine(fin);
144 strng2.ReadLine(fin2);
145 if (!strng.BeginsWith("Telescope M"))
146 {
147 cout << "WARNING - Line 3 doesn't start with 'Telescope M'." << endl;
148 cout << strng << endl;
149 }
150 }
151
152 if (version >= 200411130)
153 {
154 strng.ReadLine(fin);
155 strng2.ReadLine(fin2);
156 if (strng[0]!='#')
157 {
158 cout << "WARNING - '#' expected." << endl;
159 cout << strng << endl;
160 }
161 }
162
163 cout << " * V" << version << " " << endl;
164
165 Int_t cnt=0;
166 while (1)
167 {
168 // ===== Check for number of columns in file =====
169 // different arehucas versions provide a different number of columns:
170 // 200405050 - 200405140: 18 columns
171 // 200405180 - 200407270: 35 columns
172 // 200411130 - 200510250: 43 columns
173 // 200603300 - 200802200: 52 columns
174 // 200805190 - 200809120: 54 columns
175 // > 200812040: 55 columns
176 //
177
178 strng2.ReadLine(fin2);
179 //fill an array with the values separated by ' '
180 array=strng2.Tokenize(" ");
181 check=array->GetEntries();
182 cout << check << endl;
183 //check the number of columns for the different versions, if there is an empty line (check=0), do nothing
184 if (check != 0)
185 {
186 if ((version <= 200405140 && check != 18) ||
187 (version > 200405140 && version <= 200407270 && check != 35) ||
188 (version > 200407270 && version <= 200510250 && check != 43) ||
189 (version > 200510250 && version <= 200802200 && check != 52) ||
190 (version > 200802200 && version <= 200809120 && check != 54) ||
191 (version > 200809120 && check != 55))
192 {
193 strng.ReadLine(fin);
194 cout << "ERROR - Number of columns doesn't match number of required columns." << endl;
195 array->Delete();
196 continue;
197 }
198 }
199 array->Delete();
200
201
202 // ========== Col 1: Telescope Number =========
203 Int_t telnumber = 1; // FIXME: "NULL"?
204 if (version >=200805190)
205 {
206 strng.ReadToDelim(fin, ' ');
207 if (!fin)
208 break;
209 if (strng[0]!='M')
210 {
211 cout << "WARNING - First character is not an M." << endl;
212 cout << strng << endl;
213 strng.ReadLine(fin);
214 continue;
215 }
216 if (strng[1]!='1')
217 {
218 cout << "WARNING - Only MAGIC 1 implemented so far." << endl;
219 cout << strng << endl;
220 strng.ReadLine(fin);
221 continue;
222 }
223
224 telnumber = atoi(strng.Data()+1);
225 }
226
227 // ========== Col 2: Run Number =========
228 //Reading the line
229 //and converting some strings to ints/floats
230 strng.ReadToDelim(fin, ' ');
231 if (!fin)
232 break;
233
234 Int_t runnumber = atoi(strng.Data());
235
236 //runnumber=0 means no valid dataset
237 //-> continue
238 if (runnumber == 0)
239 {
240 strng.ReadLine(fin);
241 cout << "WARNING - Runnumber == 0" << endl;
242 cout << strng << endl;
243 continue;
244 }
245
246 // ========== Col 3: Subrun Number ========= starting with version 200805190
247 Int_t filenumber = 0; // FIXME: "NULL"?
248 if (version >=200805190)
249 {
250 strng.ReadToDelim(fin, ' ');
251 filenumber = atoi(strng.Data());
252 }
253
254 TString where = Form("fTelescopeNumber=%d AND fFileNumber=%d",
255 telnumber, filenumber);
256 if (serv.ExistStr("fRunNumber", "RunData", Form("%d", runnumber), where))
257 {
258 // FIXME: Maybe we can implement a switch to update mode?
259 cout << "WARNING - Entry M" << telnumber << ":" << runnumber << "/" << filenumber << " already existing... skipped." << endl;
260 strng.ReadLine(fin);
261 continue;
262 }
263
264 // ========== Col 4: Run Type =========
265 strng.ReadToDelim(fin, ' ');
266 if (strng.Contains("???"))
267 strng="n/a";
268
269 Int_t runtype = serv.QueryKeyOfName("RunType", strng, kFALSE);
270 if (runtype<0)
271 {
272 cout << "ERROR - RunType " << strng << " not available." << endl;
273 strng.ReadLine(fin);
274 continue;
275 }
276
277 //cout << runtype << " ";
278
279 // ========== Col 5,6: Start Time =========
280 TString startdate, starttime;
281 startdate.ReadToDelim(fin, ' ');
282 starttime.ReadToDelim(fin, ' ');
283 //cout << startdate << " " << starttime << " ";
284
285 // ========== Col 7,8: Stop Time =========
286 TString stopdate, stoptime;
287 stopdate.ReadToDelim(fin, ' ');
288 stoptime.ReadToDelim(fin, ' ');
289 //cout << stopdate << " " << stoptime << " ";
290
291 if (startdate.Contains("???"))
292 startdate="0000-00-00";
293 if (starttime.Contains("???"))
294 starttime="00:00:00";
295 if (stopdate.Contains("???"))
296 stopdate="0000-00-00";
297 if (stoptime.Contains("???"))
298 stoptime="00:00:00";
299
300 // ========== Col 9: Source Name =========
301 strng.ReadToDelim(fin, ' ');
302 if (strng.Contains("???"))
303 strng="Unavailable";
304
305 Int_t sourcekey = serv.QueryKeyOfName("Source", strng.Data());
306 if (sourcekey<0)
307 {
308 strng.ReadLine(fin);
309 continue;
310 }
311 //cout << sourcekey << " ";
312
313 // ========== Col 10,11: Local source position =========
314 strng.ReadToDelim(fin, ' ');
315 Float_t zd = atof(strng.Data());
316
317 strng.ReadToDelim(fin, ' ');
318 Float_t az = atof(strng.Data());
319
320 //cout << zd << " " << az << " ";
321
322 // ========== Col 12: Number of Events =========
323 strng.ReadToDelim(fin, ' ');
324 Int_t evtno = atoi(strng.Data());
325
326 //cout << evtno << " ";
327
328 // ========== Col 13: Project Name =========
329 strng.ReadToDelim(fin, ' ');
330 if (strng.Contains("???"))
331 strng="Unavailable";
332
333 Int_t projkey = serv.QueryKeyOfName("Project", strng);
334 if (projkey<0)
335 {
336 strng.ReadLine(fin);
337 continue;
338 }
339 //cout << projkey << " ";
340
341 // ========== Col 14: Trigger Table Name =========
342 // starting from version 200411130: Col 14,15: Trigger Table Name =========
343 strng.ReadToDelim(fin, ' ');
344 if (strng.Contains("???"))
345 strng="n/a";
346
347 Int_t l1triggerkey=1;
348 Int_t l2triggerkey=1;
349 if (version >=200411130)
350 {
351 l1triggerkey = serv.QueryKeyOfName("L1TriggerTable", strng);
352 if (l1triggerkey<0)
353 {
354 strng.ReadLine(fin);
355 continue;
356 }
357
358 strng.ReadToDelim(fin, ' ');
359 if (strng.Contains("???"))
360 strng="n/a";
361
362 l2triggerkey = serv.QueryKeyOfName("L2TriggerTable", strng);
363 if (l2triggerkey<0)
364 {
365 strng.ReadLine(fin);
366 continue;
367 }
368 }
369 else
370 {
371 Int_t c=0;
372
373 if (strng.Contains(":"))
374 c=1;
375
376 if (strng.Contains("L1_") && !(strng.Contains(":")))
377 c=2;
378
379 if (strng.Contains("n/a"))
380 c=3;
381
382 switch (c)
383 {
384 case 0:
385 {
386 l2triggerkey = serv.QueryKeyOfName("L2TriggerTable", strng);
387 if (l2triggerkey<0)
388 {
389 strng.ReadLine(fin);
390 continue;
391 }
392
393 strng="n/a";
394 l1triggerkey = 1;
395
396 break;
397 }
398 case 1:
399 {
400 TString L1TT, L2TT;
401 L2TT=strng(7,12);
402 L1TT=strng(0,6);
403
404 l1triggerkey = serv.QueryKeyOfName("L1TriggerTable", L1TT);
405 if (l1triggerkey<0)
406 {
407 strng.ReadLine(fin);
408 continue;
409 }
410
411 l2triggerkey = serv.QueryKeyOfName("L2TriggerTable", L2TT);
412 if (l2triggerkey<0)
413 {
414 strng.ReadLine(fin);
415 continue;
416 }
417
418 break;
419 }
420 case 2:
421 {
422 l1triggerkey = serv.QueryKeyOfName("L1TriggerTable", strng);
423 if (l1triggerkey<0)
424 {
425 strng.ReadLine(fin);
426 continue;
427 }
428
429 strng="n/a";
430 l2triggerkey = 1;
431
432 break;
433 }
434 case 3:
435 {
436 l1triggerkey = 1;
437 l2triggerkey = 1;
438 break;
439 }
440 default:
441 {
442 cout << "WARNING: neither L1 nor L2 Trigger table - please check what is happening." << strng << endl;
443 break;
444 }
445 }
446 }
447
448 // ========== Col 16-18: TrigRate, L2 UnPresc Rate, L2 Presc Rate ==========
449 strng.ReadToDelim(fin, ' ');
450 Float_t trigrate = atof(strng.Data());
451
452 strng.ReadToDelim(fin, ' ');
453 Float_t l2uprate = atof(strng.Data());
454
455 strng.ReadToDelim(fin, ' ');
456 Float_t l2prrate = atof(strng.Data());
457
458 // ========== Col 19,20: DaqRate, Storage Rate ==========
459 strng.ReadToDelim(fin, ' ');
460 Float_t daqrate = atof(strng.Data());
461
462 strng.ReadToDelim(fin, ' ');
463 Float_t storerate = atof(strng.Data());
464
465 // ========== Col 21: HV table =========
466 if (version==200405050 || version==200405140)
467 strng.ReadToDelim(fin, '\n');
468 else
469 strng.ReadToDelim(fin, ' ');
470 if (strng.Contains("???"))
471 strng="n/a";
472
473 Int_t hvkey = serv.QueryKeyOfName("HvSettings", strng);
474 if (hvkey<0)
475 {
476 //strng.ReadLine(fin);
477 continue;
478 }
479
480 if (version==200405180 || version==200407270)
481 strng.ReadLine(fin);
482
483 Int_t testflagkey=1;
484 Int_t lightcondkey=1;
485 Int_t dttablekey=1;
486 Int_t triggerdelaytablekey=1;
487 Int_t calibrationscriptkey=1;
488 if (version>=200411130)
489 {
490 // ========== Col 22-38: DC and HV-values, mjd =========
491 for (int i=0 ; i<17 ; i++)
492 {
493 strng.ReadToDelim(fin, ' ');
494 }
495
496 // ========== Col 39: test-flag =========
497 strng.ReadToDelim(fin, ' ');
498 if (strng.Contains("???"))
499 strng="n/a";
500
501 testflagkey = serv.QueryKeyOfName("TestFlag", strng);
502 if (testflagkey<0)
503 {
504 strng.ReadLine(fin);
505 continue;
506 }
507
508 // ========== Col 40: light conditions =========
509 strng.ReadToDelim(fin, ' ');
510 if (strng.Contains("???"))
511 strng="n/a";
512
513 lightcondkey = serv.QueryKeyOfName("LightConditions", strng);
514 if (lightcondkey<0)
515 {
516 strng.ReadLine(fin);
517 continue;
518 }
519
520 // ========== Col 41: discriminator threshold table =========
521 strng.ReadToDelim(fin, ' ');
522 if (strng.Contains("???"))
523 strng="n/a";
524
525 dttablekey = serv.QueryKeyOfName("DiscriminatorThresholdTable", strng);
526 if (dttablekey<0)
527 {
528 strng.ReadLine(fin);
529 continue;
530 }
531
532 // ========== Col 42: trigger delay table =========
533 strng.ReadToDelim(fin, ' ');
534 if (strng.Contains("???"))
535 strng="n/a";
536
537 triggerdelaytablekey = serv.QueryKeyOfName("TriggerDelayTable", strng);
538 if (triggerdelaytablekey<0)
539 {
540 strng.ReadLine(fin);
541 continue;
542 }
543
544 // ========== Col 43,44: Telescope RA and Dec sent to drive =========
545 strng.ReadToDelim(fin, ' ');
546 strng.ReadToDelim(fin, ' ');
547
548 // ========== Col 45: Calibration Script =========
549 if (version>=200411130 && version<=200510250)
550 strng.ReadToDelim(fin, '\n');
551 else
552 strng.ReadToDelim(fin, ' ');
553 if (strng.Contains("???"))
554 strng="n/a";
555
556 calibrationscriptkey = serv.QueryKeyOfName("CalibrationScript", strng);
557 if (calibrationscriptkey<0)
558 {
559 strng.ReadLine(fin);
560 continue;
561 }
562
563 }
564
565 Int_t observationmodekey=1;
566 if (version>=200603300)
567 {
568 // ========== Col 46: Observation Mode =========
569 strng.ReadToDelim(fin, ' ');
570 if (strng.Contains("???"))
571 strng="n/a";
572
573 observationmodekey = serv.QueryKeyOfName("ObservationMode", strng);
574 if (observationmodekey<0)
575 {
576 strng.ReadLine(fin);
577 continue;
578 }
579
580 // ========== Col 47-54: Source RA and Dec, DT's and IPR =========
581 for (int i=0 ; i<7 ; i++)
582 {
583 strng.ReadToDelim(fin, ' ');
584 }
585 if (version<=200809120)
586 strng.ReadToDelim(fin, '\n');
587 else
588 strng.ReadToDelim(fin, ' ');
589 }
590
591 Int_t sumtriggerflagkey=1;
592 if (version>=200812040)
593 {
594 // ========= Col 55: SumTrigger flag =========
595 strng.ReadToDelim(fin, '\n');
596 if (strng.Contains("???"))
597 strng="n/a";
598
599 sumtriggerflagkey = serv.QueryKeyOfName("SumTriggerFlag", strng);
600 if (sumtriggerflagkey<0)
601 {
602 strng.ReadLine(fin);
603 continue;
604 }
605
606 }
607
608
609 // ================================================================
610 // ========== Data read from file now access the database =========
611 // ================================================================
612
613 //assemble the query that is needed to insert the values of this run
614 TString query;
615 query += Form("fTelescopeNumber=%d, ", telnumber);
616 query += Form("fRunNumber=%d, ", runnumber);
617 query += Form("fFileNumber=%d, ", filenumber);
618 query += Form("fRunTypeKEY=%d, ", runtype);
619 query += Form("fProjectKEY=%d, ", projkey);
620 query += Form("fSourceKEY=%d, ", sourcekey);
621 query += Form("fNumEvents=%d, ", evtno);
622 query += Form("fRunStart=\"%s %s\", ", startdate.Data(), starttime.Data());
623 query += Form("fRunStop=\"%s %s\", ", stopdate.Data(), stoptime.Data());
624 query += Form("fL1TriggerTableKEY=%d, ", l1triggerkey);
625 query += Form("fL2TriggerTableKEY=%d, ", l2triggerkey);
626 query += Form("fTestFlagKEY=%d, ", testflagkey);
627 query += Form("fCalibrationScriptKEY=%d, ", calibrationscriptkey);
628 query += Form("fTriggerDelayTableKEY=%d, ", triggerdelaytablekey);
629 query += Form("fDiscriminatorThresholdTableKEY=%d, ", dttablekey);
630 query += Form("fLightConditionsKEY=%d, ", lightcondkey);
631 query += Form("fHvSettingsKEY=%d, ", hvkey);
632 query += Form("fObservationModeKEY=%d, ", observationmodekey);
633 query += Form("fSumTriggerFlagKEY=%d, ", sumtriggerflagkey);
634 if (!TMath::IsNaN(zd) && TMath::Finite(zd))
635 query += Form("fZenithDistance=%d, ", TMath::Nint(zd));
636 if (!TMath::IsNaN(az) && TMath::Finite(az))
637 query += Form("fAzimuth=%d, ", TMath::Nint(az));
638 if (!TMath::IsNaN(storerate) && TMath::Finite(storerate))
639 query += Form("fDaqStoreRate=%d, ", TMath::Nint(storerate));
640 if (!TMath::IsNaN(daqrate) && TMath::Finite(daqrate))
641 query += Form("fDaqTriggerRate=%d, ", TMath::Nint(daqrate));
642 if (!TMath::IsNaN(trigrate) && TMath::Finite(trigrate))
643 query += Form("fMeanTriggerRate=%d, ", TMath::Nint(trigrate));
644 if (!TMath::IsNaN(l2prrate) && TMath::Finite(l2prrate))
645 query += Form("fL2RatePresc=%d, ", TMath::Nint(l2prrate));
646 if (!TMath::IsNaN(l2uprate) && TMath::Finite(l2uprate))
647 query += Form("fL2RateUnpresc=%d, ", TMath::Nint(l2uprate));
648 query += "fMagicNumberKEY=1, fExcludedFDAKEY=1";
649
650 cnt++;
651
652 //send query, add dataset to DB
653 if (serv.Insert("RunData", query)==kFALSE)
654 return -1;
655
656 TString query2=Form("fTelescopeNumber=%d, fRunNumber=%d, fFileNumber=%d, "
657 "fPriority=%d, fTimingCorrection='1970-01-01 00:00:00', fCompmux='1970-01-01 00:00:00'",
658 telnumber, runnumber, filenumber, runnumber);
659 if (testflagkey==3)
660 query2+=" , fDataCheckDone='1970-01-01 00:00:00'";
661
662 //create entry in table RunProcessStatus for this runnumber
663 if (serv.Insert("RunProcessStatus", query2)==kFALSE)
664 return -1;
665 }
666
667 return cnt;
668}
669
670// This tool will work from Period017 (2004_05_17) on...
671int filldotrun(const TString path="/home/lapalma/transfer/ccdata", Bool_t dummy=kTRUE)
672{
673 MSQLMagic serv("sql.rc");
674 if (!serv.IsConnected())
675 {
676 cout << "ERROR - Connection to database failed." << endl;
677 return 0;
678 }
679
680 cout << "filldotrun" << endl;
681 cout << "----------" << endl;
682 cout << endl;
683 cout << "Connected to " << serv.GetName() << endl;
684 cout << "Search Path: " << path << endl;
685 cout << endl;
686
687 serv.SetIsDummy(dummy);
688
689 if (path.EndsWith(".run"))
690 {
691 cout << path(TRegexp("CC_.*.run", kFALSE)) << flush;
692 Int_t n = insert(serv, dummy, path);
693 cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
694
695 return n<0 ? 2 : 1;
696 }
697
698 MDirIter Next(path, "CC_*.run", -1);
699 while (1)
700 {
701 TString name = Next();
702 if (name.IsNull())
703 break;
704
705 cout << " * " << name(TRegexp("CC_.*.run", kFALSE)) << endl;
706 Int_t n = insert(serv, dummy, name);
707 cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
708
709 if (n<0)
710 return 2;
711 }
712
713 return 1;
714}
Note: See TracBrowser for help on using the repository browser.