source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 7234

Last change on this file since 7234 was 7232, checked in by Daniela Dorner, 21 years ago
*** empty log message ***
File size: 20.4 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2004
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38//
39/////////////////////////////////////////////////////////////////////////////
40#include <iostream>
41#include <iomanip>
42#include <fstream>
43
44#include <MSQLServer.h>
45#include <TSQLRow.h>
46#include <TSQLResult.h>
47
48#include <TEnv.h>
49#include <TMath.h>
50#include <TExMap.h>
51#include <TArrayI.h>
52#include <TRegexp.h>
53#include <TSystem.h>
54
55#include <MTime.h>
56#include <MDirIter.h>
57
58using namespace std;
59
60int debug = 0;
61
62Bool_t DeleteSequence(MSQLServer &serv, TString datapath, TString sequpath, Int_t sequ, Bool_t dummy)
63{
64 TString query1(Form("DELETE FROM Calibration WHERE fSequenceFirst=%d", sequ));
65 TString query2(Form("DELETE FROM Star WHERE fSequenceFirst=%d", sequ));
66 TString query3(Form("DELETE FROM SequenceProcessStatus WHERE fSequenceFirst=%d", sequ));
67 TString query4(Form("UPDATE RunData SET fSequenceFirst=0 WHERE fSequenceFirst=%d", sequ));
68 TString query5(Form("DELETE FROM Sequences WHERE fSequenceFirst=%d AND fManuallyChangedKEY=1", sequ));
69
70 TString fname(Form("%s/%04d/sequence%08d.txt", sequpath.Data(),sequ/10000, sequ));
71 TString command(Form("rm -r %s/callisto/%04d/%08d/", datapath.Data(), sequ/10000, sequ));
72 TString command2(Form("rm -r %s/star/%04d/%08d/", datapath.Data(), sequ/10000, sequ));
73
74 if (dummy)
75 {
76 cout << "not using dummy=kTRUE the following commands would be executed: " << endl;
77 cout << "queries: " << endl;
78 cout << query1 << endl;
79 cout << query2 << endl;
80 cout << query3 << endl;
81 cout << query4 << endl;
82 cout << query5 << endl;
83 cout << "removing files:" << endl;
84 cout << "unlink " << fname << endl;
85 cout << command << endl;
86 cout << command2 << endl;
87 return kTRUE;
88 }
89
90 TSQLResult *res = serv.Query(query1);
91 if (!res)
92 return kFALSE;
93 delete res;
94
95 res = serv.Query(query2);
96 if (!res)
97 return kFALSE;
98 delete res;
99
100 res = serv.Query(query3);
101 if (!res)
102 return kFALSE;
103 delete res;
104
105 res = serv.Query(query4);
106 if (!res)
107 return kFALSE;
108 delete res;
109
110 res = serv.Query(query5);
111 if (!res)
112 return kFALSE;
113 delete res;
114
115 gSystem->Unlink(fname);
116
117 gSystem->Exec(command);
118 gSystem->Exec(command2);
119
120 return kTRUE;
121}
122
123Int_t DoCheck(TSQLResult &res)
124{
125 TArrayI data(5);
126 Int_t n = 0;
127
128 TSQLRow *row=0;
129 while ((row=res.Next()))
130 {
131 n++;
132
133 if (data[0]==0)
134 {
135 for (int i=0; i<data.GetSize(); i++)
136 data[i] = atoi((*row)[i]);
137 continue;
138 }
139
140 for (int i=1; i<data.GetSize(); i++)
141 {
142 if (data[i] != atoi((*row)[i]))
143 return i+1;
144 }
145 }
146 return n==0 ? 0 : -1;
147}
148
149Bool_t CheckRuns(MSQLServer &serv, Int_t from, Int_t to, Int_t type)
150{
151 TString query("SELECT fRunNumber, fL1TriggerTableKEY, fL2TriggerTableKEY,"
152 " fCalibrationScriptKEY, fProjectKEY FROM RunData");
153 query += Form(" WHERE fRunTypeKEY=%d AND fExcludedFDAKEY=1 AND "
154 " (fRunNumber BETWEEN %d AND %d)"
155 " ORDER BY fRunNumber", type, from, to);
156
157 TSQLResult *res = serv.Query(query);
158 if (!res)
159 return kFALSE;
160
161 Int_t rc = DoCheck(*res);
162 delete res;
163
164 switch (rc)
165 {
166 case 0: cout << "ERROR - No runs found for check!" << endl; break;
167 case 1: cout << "ERROR - fRunNumber doesn't match!" << endl; break;
168 case 2: cout << "ERROR - fL1TriggerTableKEY doesn't match!" << endl; break;
169 case 3: cout << "ERROR - fL2TriggerTableKEY doesn't match!" << endl; break;
170 case 4: cout << "ERROR - fCalibrationScriptKEY doesn't match!" << endl; break;
171 case 5: cout << "ERROR - fProjectKEY doesn't match!" << endl; break;
172 }
173
174 return rc<0;
175}
176
177Bool_t CheckSequence(MSQLServer &serv, TString datapath, TString sequpath, Int_t from, Int_t to, Bool_t dummy)
178{
179 Bool_t rc=kFALSE;
180
181 TString query("SELECT fSequenceFirst, count(*) FROM RunData ");
182 query += Form(" WHERE fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
183 " GROUP BY fSequenceFirst", from, to);
184
185 TSQLResult *res = serv.Query(query);
186 if (!res)
187 return kFALSE;
188
189 Int_t count=0;
190 Int_t nor=0;
191 TArrayI sequences;
192
193 TSQLRow *row=0;
194 while ((row=res->Next()))
195 {
196 sequences.Set(count+1);
197 sequences.AddAt(atoi((*row)[0]), count);
198// cout << "sequence: " << sequences.At(count) << endl;
199 nor=atoi((*row)[1]);
200 count++;
201 }
202
203 delete res;
204
205 if (sequences.GetSize()>1)
206 {
207 for (Int_t i=0;i<sequences.GetSize();i++)
208 {
209 rc=kTRUE;
210 if (sequences.At(i)!=0)
211 {
212 cout << "deleting sequence " << sequences.At(i) << "... <" << i << ">" << endl;
213 if(!DeleteSequence(serv, datapath, sequpath, sequences.At(i), dummy))
214 rc=kFALSE;
215 }
216 }
217 return rc;
218 }
219
220 if (sequences.At(0)==0)
221 return kTRUE;
222
223
224 query="SELECT fRunNumber FROM RunData ";
225 query += Form(" WHERE fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
226 " ORDER BY fRunNumber", from, to);
227
228 res = serv.Query(query);
229 if (!res)
230 return kFALSE;
231
232 TArrayI runs;
233 count=0;
234 row=0;
235 while ((row=res->Next()))
236 {
237 runs.Set(count+1);
238 runs.AddAt(atoi((*row)[0]), count);
239 count++;
240 }
241
242 delete res;
243
244
245 if (nor==runs.GetSize() && sequences.GetSize()==1)
246 {
247 query="SELECT fRunNumber FROM RunData ";
248 query += Form(" WHERE fSequenceFirst=%d ", sequences.At(0));
249
250 res = serv.Query(query);
251 if (!res)
252 return kFALSE;
253
254 TArrayI oldruns;
255 count=0;
256 row=0;
257 while ((row=res->Next()))
258 {
259 oldruns.Set(count+1);
260 oldruns.AddAt(atoi((*row)[0]), count);
261 count++;
262 }
263
264 if (runs.GetSize()!=oldruns.GetSize())
265 {
266 cout << "different sequsizes -> deleting sequence " << sequences.At(0) << endl;
267 return DeleteSequence(serv, datapath, sequpath, sequences.At(0), dummy);
268 }
269
270 for (Int_t i=0; i<runs.GetSize(); i++)
271 {
272 if (runs.At(i)==oldruns.At(i))
273 continue;
274 cout << "different runs -> deleting sequence " << sequences.At(0) << endl;
275 return DeleteSequence(serv, datapath, sequpath, sequences.At(0), dummy);
276 }
277
278 delete res;
279 }
280
281 cout << "same sequence -> nothing to do..." << endl;
282 return rc;
283}
284
285Bool_t InsertSequence(MSQLServer &serv, Int_t from, Int_t to)
286{
287
288 // ========== Request number of events ==========
289 TString query("SELECT SUM(fNumEvents), "
290 " SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
291 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
292 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ");
293 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
294 query += " MIN(fAzimuth), MAX(fAzimuth) ";
295 query += Form(" FROM RunData WHERE fRunTypeKEY=2 AND "
296 " (fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
297 from, to);
298
299 TSQLResult *res = serv.Query(query);
300 if (!res)
301 return kFALSE;
302
303 TSQLRow *row = res->Next();
304 if (!row || !(*row)[0])
305 {
306 cout << "ERROR - No result from query: " << query << endl;
307 return kFALSE;
308 }
309
310 TString nevts = (*row)[0];
311 TString secs = (*row)[1];
312 TString zdmin = (*row)[2];
313 TString zdmax = (*row)[3];
314 TString azmin = (*row)[4];
315 TString azmax = (*row)[5];
316
317 delete res;
318
319 // ========== Request start time of sequence ==========
320 query = Form("SELECT fRunStart FROM RunData WHERE fRunNumber=%d AND fExcludedFDAKEY=1", from);
321
322 res = serv.Query(query);
323 if (!res)
324 return kFALSE;
325
326 row = res->Next();
327 if (!row || !(*row)[0])
328 {
329 cout << "ERROR - No result from query: " << query << endl;
330 return kFALSE;
331 }
332
333 TString start((*row)[0]);
334
335 delete res;
336
337 // ========== Request data of sequence ==========
338 query = Form("SELECT fSourceKEY, fProjectKEY, "
339 " fL1TriggerTableKEY, fL1TriggerTableKEY,"
340 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
341 " fTriggerDelayTableKEY, fLightConditionsKEY, fTestFlagKEY"
342 " FROM RunData"
343 " WHERE fRunTypeKEY=2 AND fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
344 " LIMIT 1", from, to);
345
346 res = serv.Query(query);
347 if (!res)
348 return kFALSE;
349
350 row = res->Next();
351 if (!row)
352 {
353 cout << "ERROR - No result from query: " << query << endl;
354 return kFALSE;
355 }
356
357 TString query1("INSERT Sequences SET");
358 query1+=Form(" fSequenceFirst=%d, fSequenceLast=%d,", from, to);
359 query1+=Form(" fProjectKEY=%s,", (*row)[0]);
360 query1+=Form(" fSourceKEY=%s,", (*row)[1]);
361 query1+=Form(" fNumEvents=%s,", nevts.Data());
362 query1+=Form(" fRunTime=%s,", secs.Data());
363 query1+=Form(" fRunStart=\"%s\",", start.Data());
364 query1+=Form(" fZenithDistanceMin=%s,", zdmin.Data());
365 query1+=Form(" fZenithDistanceMax=%s,", zdmax.Data());
366 query1+=Form(" fAzimuthMin=%s,", azmin.Data());
367 query1+=Form(" fAzimuthMax=%s,", azmax.Data());
368 query1+=Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
369 query1+=Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
370 query1+=Form(" fHvSettingsKEY=%s,", (*row)[4]);
371 query1+=Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
372 query1+=Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
373 query1+=Form(" fLightConditionsKEY=%s,", (*row)[7]);
374 query1+=Form(" fTestFlagKEY=%s, fManuallyChangedKEY=1", (*row)[8]);
375
376
377 TString query2 = Form("UPDATE RunData SET fSequenceFirst=%d WHERE"
378 " (fRunNumber BETWEEN %d AND %d) AND"
379 " (fRunTypeKEY BETWEEN 2 AND 4) AND"
380 " fSourceKEY=%d AND fHvSettingsKEY=%s AND fExcludedFDAKEY=1",
381 from, from, to, (*row)[1], (*row)[4]);
382
383 TString query3 = Form("INSERT SequenceProcessStatus SET fSequenceFirst=%d ", from);
384
385 delete res;
386
387 res = serv.Query(query1);
388 if (!res)
389 return kFALSE;
390 delete res;
391
392 res = serv.Query(query2);
393 if (!res)
394 return kFALSE;
395 delete res;
396
397 res = serv.Query(query3);
398 if (!res)
399 return kFALSE;
400 delete res;
401
402 return kTRUE;
403}
404
405Bool_t NewSequence(MSQLServer &serv, TString datapath, TString sequpath, Int_t from, Int_t to, Bool_t dummy)
406{
407 cout << "Found Sequence (" << from << ", " << to << ") ... checking runs..." << flush;
408
409 if (!CheckRuns(serv, from, to, 2))
410 {
411 cout << "Warning - Found inconsistency in data-runs (" << from << ", " << to << ")" << endl;
412 //sequence is not built, but kTRUE is returned, to allow
413 //the automatic processing of the other sequences of this day
414 return kTRUE;
415 }
416 if (!CheckRuns(serv, from, to, 3))
417 {
418 cout << "Warning - Found inconsistency in ped-runs (" << from << ", " << to << ")" << endl;
419 //sequence is not built, but kTRUE is returned, to allow
420 //the automatic processing of the other sequences of this day
421 return kTRUE;
422 }
423
424 cout << "ok." << endl;
425
426
427 cout << "checking Sequence..." << endl;
428
429 if (!CheckSequence(serv, datapath, sequpath, from, to, dummy))
430 {
431 cout << " inserting sequence not necessary" << endl;
432 return kTRUE;
433 }
434
435
436 if (dummy)
437 return kTRUE;
438
439 Bool_t rc = InsertSequence(serv, from, to);
440 if (!rc)
441 cout << "InsertSequence failed!" << endl;
442
443 return rc;
444}
445
446Bool_t Process(MSQLServer &serv, TString datapath, TString sequpath, Int_t from, Int_t to, Bool_t dummy)
447{
448
449 TString query(Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
450 " FROM RunData"
451 " WHERE fRunNumber BETWEEN %d AND %d AND "
452 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
453 " ORDER BY fRunNumber", from, to));
454
455 TSQLResult *res = serv.Query(query);
456 if (!res)
457 return kFALSE;
458
459 TExMap map;
460
461 Int_t start=0;
462 Int_t stop=0;
463 Int_t last=0;
464 Int_t first=0;
465
466 MTime lasttime;
467
468 TSQLRow *row=0;
469
470 enum { UNKNOWN, PED=3, CAL=4, DATA=2 };
471 Char_t status = UNKNOWN;
472
473 Int_t nblocks = 0;
474
475 while ((row=res->Next()))
476 {
477 if (!(*row)[1])
478 continue;
479
480 if (start==0)
481 {
482 first = atoi((*row)[0]);
483 if (debug)
484 cout << "First Run: " << first << endl;
485 }
486
487 switch (atoi((*row)[1]))
488 {
489 case CAL: // ---------- CALIBRATION ----------
490 if (status!=CAL)
491 {
492 start = stop = atoi((*row)[0]);
493 if (!(*row)[2])
494 cout << "No time available... skipped." << endl;
495 else
496 {
497 MTime *tm = new MTime;
498 tm->SetSqlDateTime((*row)[2]);
499 map.Add((ULong_t)map.GetSize(), (Long_t)tm, (Long_t)nblocks);
500 }
501 }
502 status = CAL;
503 break;
504 default:
505 if (status==CAL)
506 {
507 MTime *tm = new MTime(lasttime);
508 map.Add((ULong_t)map.GetSize(), (Long_t)tm, (Long_t)nblocks);
509
510 stop = last;
511 nblocks++;
512 if (debug)
513 cout << "Cal Block #" << nblocks << " from " << start << " to " << last << endl;
514 }
515 status = UNKNOWN;
516 break;
517 }
518 last = atoi((*row)[0]);
519 lasttime.SetSqlDateTime((*row)[3]);
520 }
521 if (status==CAL)
522 {
523 stop = last;
524 nblocks++;
525 if (debug)
526 cout << "Cal Block #" << nblocks << " from " << start << " to " << stop << endl;
527 }
528
529 if (debug)
530 cout << "Last Run: " << last << endl;
531 delete res;
532
533 if (debug)
534 cout << "Found " << nblocks << " calibration blocks" << endl;
535
536 res = serv.Query(query);
537 if (!res)
538 return kFALSE;
539
540 Int_t n = -1;
541
542 Bool_t rc = kTRUE;
543
544 start = first;
545 while ((row=res->Next()))
546 {
547 if (!(*row)[1])
548 continue;
549
550 MTime tstart, tstop;
551 tstart.SetSqlDateTime((*row)[2]);
552 tstop.SetSqlDateTime((*row)[3]);
553
554 MTime min;
555 Int_t nmin = -1;
556 Double_t dmin = 1e35;
557
558 Long_t key, val;
559 TExMapIter nmap(&map);
560 while (nmap.Next(key, val))
561 {
562 MTime *t = (MTime*)key;
563
564 if (nmin==-1)
565 {
566 nmin = val;
567 min = *(MTime*)key;
568 dmin = fabs((Double_t)*t-(Double_t)tstart);
569 }
570
571 if (fabs((Double_t)*t-(Double_t)tstart) < dmin)
572 {
573 nmin = val;
574 dmin = fabs((Double_t)*t-(Double_t)tstart);
575 min = *t;
576 }
577 if (fabs((Double_t)*t-(Double_t)tstop) < dmin)
578 {
579 nmin = val;
580 dmin = fabs((Double_t)*t-(Double_t)tstop);
581 min = *t;
582 }
583 }
584
585 if (n!=nmin)
586 {
587 if (n!=-1)
588 {
589 if (!NewSequence(serv, datapath, sequpath, start, last, dummy))
590 {
591 rc = kFALSE;
592 //continue;
593 }
594 }
595 n = nmin;
596 start = atoi((*row)[0]);
597 }
598 last = atoi((*row)[0]);
599 }
600
601 delete res;
602
603 if (n!=-1 && start!=last)
604 {
605 if (!NewSequence(serv, datapath, sequpath, start, last, dummy))
606 rc = kFALSE;
607 }
608
609 if (debug)
610 cout << endl;
611
612 return rc;
613}
614
615
616
617int buildsequenceentries(TString day, TString datapath, TString sequpath, Bool_t dummy=kTRUE)
618{
619 TEnv env("sql.rc");
620
621 MSQLServer serv(env);
622 if (!serv.IsConnected())
623 {
624 cout << "ERROR - Connection to database failed." << endl;
625 return 0;
626 }
627
628 cout << "buildsequences" << endl;
629 cout << "--------------" << endl;
630 cout << endl;
631 cout << "Connected to " << serv.GetName() << endl;
632 cout << "Night of sunrise at: " << day << endl;
633 cout << endl;
634
635 day += " 13:00:00";
636 const TString cond(Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\")",
637 day.Data(), day.Data()));
638
639 TString query(Form("SELECT fRunNumber, fSourceKEY, fProjectKEY, fHvSettingsKEY, fLightConditionsKEY, fDiscriminatorThresholdTableKEY, fTriggerDelayTableKEY FROM RunData WHERE %s AND fExcludedFDAKEY=1 order by fRunNumber", cond.Data()));
640
641 TSQLResult *res = serv.Query(query);
642 if (!res)
643 return 0;
644
645 TString keys[6]= { "NULL", "NULL", "NULL", "NULL", "NULL", "NULL" };
646 TString stop = "NULL";
647 TString runstart = "NULL";
648 TString runstop = "NULL";
649 Int_t count = 0;
650 TExMap blocks;
651 Int_t runbegin;
652 Int_t runend;
653
654 TSQLRow *row=0;
655 while ((row=res->Next()))
656 {
657 if (count==0)
658 {
659 for (Int_t i=1 ; i<7 ; i++)
660 keys[i-1]=(*row)[i];
661 runstart=(*row)[0];
662 }
663
664 for (Int_t i=1 ; i<7 ; i++)
665 {
666 runbegin=atoi(runstart.Data());
667 runend=atoi(runstop.Data());
668 if (i==2 && runbegin>20100 && runend<45100)
669 continue;
670
671 TString value=(*row)[i];
672 TString key=keys[i-1];
673 if (!value.CompareTo(key))
674 continue;
675
676 keys[i-1]=value;
677 //hier einfuellen
678 blocks.Add((ULong_t)blocks.GetSize(), (Long_t)runbegin, (Long_t)runend);
679 runstart=(*row)[0];
680 for (Int_t i=1 ; i<7 ; i++)
681 keys[i-1]=(*row)[i];
682 break;
683 }
684 runstop=(*row)[0];
685 count++;
686 }
687
688 //und hier einfuellen (letzter wert)
689 runbegin=atoi(runstart.Data());
690 runend=atoi(runstop.Data());
691 blocks.Add((ULong_t)blocks.GetSize(), (Long_t)runbegin, (Long_t)runend);
692
693
694 Bool_t rc = kTRUE;
695
696 Long_t key, val;
697 TExMapIter nblocks(&blocks);
698 while (nblocks.Next(key, val))
699 {
700 Int_t runstart2 = (Int_t)key;
701 Int_t runstop2 = (Int_t)val;
702 cout << endl << "datablock from " << runstart2 << " to " << runstop2 << endl;
703
704 if (!Process(serv, datapath, sequpath, runstart2, runstop2, dummy))
705 rc = kFALSE;
706
707 }
708 return rc ? 1 : 0;
709}
710
711
712int buildsequenceentries(TString datapath, TString sequpath, Bool_t dummy=kTRUE)
713{
714 TEnv env("sql.rc");
715
716 MSQLServer serv(env);
717 if (!serv.IsConnected())
718 {
719 cout << "ERROR - Connection to database failed." << endl;
720 return 0;
721 }
722
723 TString query="SELECT fDate FROM SequenceBuildStatus";
724
725 TSQLResult *res = serv.Query(query);
726 if (!res)
727 return 0;
728
729 TSQLRow *row=0;
730 while ((row=res->Next()))
731 {
732 cout << "date: " << (*row)[0] << endl;
733 buildsequenceentries((*row)[0], dummy);
734 }
735
736 return 1;
737}
Note: See TracBrowser for help on using the repository browser.