source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 7228

Last change on this file since 7228 was 7215, checked in by Daniela Dorner, 19 years ago
*** empty log message ***
File size: 20.0 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2004
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - sorts the runs by source
34// - groups the runs to sequences such that each run belongs
35// to the nearest (in time) calibration run
36//
37// apart from that the runs in one sequence must have the same
38// hv-settings, trigger tables, light conditions, DT tables,
39// trigger delay tables and testflag
40//
41/////////////////////////////////////////////////////////////////////////////
42#include <iostream>
43#include <iomanip>
44#include <fstream>
45
46#include <MSQLServer.h>
47#include <TSQLRow.h>
48#include <TSQLResult.h>
49
50#include <TEnv.h>
51#include <TMath.h>
52#include <TExMap.h>
53#include <TArrayI.h>
54#include <TRegexp.h>
55#include <TSystem.h>
56
57#include <MTime.h>
58#include <MDirIter.h>
59
60using namespace std;
61
62int debug = 0;
63
64Bool_t DeleteSequence(MSQLServer &serv, Int_t sequ, Bool_t dummy)
65{
66 TString query1(Form("DELETE FROM Calibration WHERE fSequenceFirst=%d", sequ));
67 TString query2(Form("DELETE FROM Star WHERE fSequenceFirst=%d", sequ));
68 TString query3(Form("DELETE FROM SequenceProcessStatus WHERE fSequenceFirst=%d", sequ));
69 TString query4(Form("UPDATE RunData SET fSequenceFirst=0 WHERE fSequenceFirst=%d", sequ));
70 TString query5(Form("DELETE FROM Sequences WHERE fSequenceFirst=%d AND fManuallyChangedKEY=1", sequ));
71
72 TString fname(Form("/mnt/stk01/sequences/%04d/sequence%08d.txt", sequ/10000, sequ));
73 TString command(Form("rm -r /magic/data/callisto/%04d/%08d/", sequ/10000, sequ));
74 TString command2(Form("rm -r /magic/data/star/%04d/%08d/", sequ/10000, sequ));
75
76 if (dummy)
77 {
78 cout << "not using dummy=kTRUE the following commands would be executed: " << endl;
79 cout << "queries: " << endl;
80 cout << query1 << endl;
81 cout << query2 << endl;
82 cout << query3 << endl;
83 cout << query4 << endl;
84 cout << query5 << endl;
85 cout << "removing files:" << endl;
86 cout << "unlink " << fname << endl;
87 cout << command << endl;
88 cout << command2 << endl;
89 return kTRUE;
90 }
91
92 TSQLResult *res = serv.Query(query1);
93 if (!res)
94 return kFALSE;
95 delete res;
96
97 res = serv.Query(query2);
98 if (!res)
99 return kFALSE;
100 delete res;
101
102 res = serv.Query(query3);
103 if (!res)
104 return kFALSE;
105 delete res;
106
107 res = serv.Query(query4);
108 if (!res)
109 return kFALSE;
110 delete res;
111
112 res = serv.Query(query5);
113 if (!res)
114 return kFALSE;
115 delete res;
116
117 gSystem->Unlink(fname);
118
119 gSystem->Exec(command);
120 gSystem->Exec(command2);
121
122 return kTRUE;
123}
124
125Int_t DoCheck(TSQLResult &res)
126{
127 TArrayI data(5);
128 Int_t n = 0;
129
130 TSQLRow *row=0;
131 while ((row=res.Next()))
132 {
133 n++;
134
135 if (data[0]==0)
136 {
137 for (int i=0; i<data.GetSize(); i++)
138 data[i] = atoi((*row)[i]);
139 continue;
140 }
141
142 for (int i=1; i<data.GetSize(); i++)
143 {
144 if (data[i] != atoi((*row)[i]))
145 return i+1;
146 }
147 }
148 return n==0 ? 0 : -1;
149}
150
151Bool_t CheckRuns(MSQLServer &serv, Int_t from, Int_t to, Int_t type)
152{
153 TString query("SELECT fRunNumber, fL1TriggerTableKEY, fL2TriggerTableKEY,"
154 " fCalibrationScriptKEY, fProjectKEY FROM RunData");
155 query += Form(" WHERE fRunTypeKEY=%d AND fExcludedFDAKEY=1 AND "
156 " (fRunNumber BETWEEN %d AND %d)"
157 " ORDER BY fRunNumber", type, from, to);
158
159 TSQLResult *res = serv.Query(query);
160 if (!res)
161 return kFALSE;
162
163 Int_t rc = DoCheck(*res);
164 delete res;
165
166 switch (rc)
167 {
168 case 0: cout << "ERROR - No runs found for check!" << endl; break;
169 case 1: cout << "ERROR - fRunNumber doesn't match!" << endl; break;
170 case 2: cout << "ERROR - fL1TriggerTableKEY doesn't match!" << endl; break;
171 case 3: cout << "ERROR - fL2TriggerTableKEY doesn't match!" << endl; break;
172 case 4: cout << "ERROR - fCalibrationScriptKEY doesn't match!" << endl; break;
173 case 5: cout << "ERROR - fProjectKEY doesn't match!" << endl; break;
174 }
175
176 return rc<0;
177}
178
179Bool_t CheckSequence(MSQLServer &serv, Int_t from, Int_t to, Bool_t dummy)
180{
181 Bool_t rc=kFALSE;
182
183 TString query("SELECT fSequenceFirst, count(*) FROM RunData ");
184 query += Form(" WHERE fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
185 " GROUP BY fSequenceFirst", from, to);
186
187 TSQLResult *res = serv.Query(query);
188 if (!res)
189 return kFALSE;
190
191 Int_t count=0;
192 Int_t nor=0;
193 TArrayI sequences;
194
195 TSQLRow *row=0;
196 while ((row=res->Next()))
197 {
198 sequences.Set(count+1);
199 sequences.AddAt(atoi((*row)[0]), count);
200// cout << "sequence: " << sequences.At(count) << endl;
201 nor=atoi((*row)[1]);
202 count++;
203 }
204
205 delete res;
206
207 if (sequences.GetSize()>1)
208 {
209 for (Int_t i=0;i<sequences.GetSize();i++)
210 {
211 rc=kTRUE;
212 if (sequences.At(i)!=0)
213 {
214 cout << "deleting sequence " << sequences.At(i) << "... <" << i << ">" << endl;
215 if(!DeleteSequence(serv, sequences.At(i), dummy))
216 rc=kFALSE;
217 }
218 }
219 return rc;
220 }
221
222 if (sequences.At(0)==0)
223 return kTRUE;
224
225
226 query="SELECT fRunNumber FROM RunData ";
227 query += Form(" WHERE fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
228 " ORDER BY fRunNumber", from, to);
229
230 res = serv.Query(query);
231 if (!res)
232 return kFALSE;
233
234 TArrayI runs;
235 count=0;
236 row=0;
237 while ((row=res->Next()))
238 {
239 runs.Set(count+1);
240 runs.AddAt(atoi((*row)[0]), count);
241 count++;
242 }
243
244 delete res;
245
246
247 if (nor==runs.GetSize() && sequences.GetSize()==1)
248 {
249 query="SELECT fRunNumber FROM RunData ";
250 query += Form(" WHERE fSequenceFirst=%d ", sequences.At(0));
251
252 res = serv.Query(query);
253 if (!res)
254 return kFALSE;
255
256 TArrayI oldruns;
257 count=0;
258 row=0;
259 while ((row=res->Next()))
260 {
261 oldruns.Set(count+1);
262 oldruns.AddAt(atoi((*row)[0]), count);
263 count++;
264 }
265
266 if (runs.GetSize()!=oldruns.GetSize())
267 {
268 cout << "different sequsizes -> deleting sequence " << sequences.At(0) << endl;
269 return DeleteSequence(serv, sequences.At(0), dummy);
270 }
271
272 for (Int_t i=0; i<runs.GetSize(); i++)
273 {
274 if (runs.At(i)==oldruns.At(i))
275 continue;
276 cout << "different runs -> deleting sequence " << sequences.At(0) << endl;
277 return DeleteSequence(serv, sequences.At(0), dummy);
278 }
279
280 delete res;
281 }
282
283 cout << "same sequence -> nothing to do..." << endl;
284 return rc;
285}
286
287Bool_t InsertSequence(MSQLServer &serv, Int_t from, Int_t to)
288{
289
290 // ========== Request number of events ==========
291 TString query("SELECT SUM(fNumEvents), "
292 " SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
293 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
294 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ");
295 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
296 query += " MIN(fAzimuth), MAX(fAzimuth) ";
297 query += Form(" FROM RunData WHERE fRunTypeKEY=2 AND "
298 " (fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
299 from, to);
300
301 TSQLResult *res = serv.Query(query);
302 if (!res)
303 return kFALSE;
304
305 TSQLRow *row = res->Next();
306 if (!row || !(*row)[0])
307 {
308 cout << "ERROR - No result from query: " << query << endl;
309 return kFALSE;
310 }
311
312 TString nevts = (*row)[0];
313 TString secs = (*row)[1];
314 TString zdmin = (*row)[2];
315 TString zdmax = (*row)[3];
316 TString azmin = (*row)[4];
317 TString azmax = (*row)[5];
318
319 delete res;
320
321 // ========== Request start time of sequence ==========
322 query = Form("SELECT fRunStart FROM RunData WHERE fRunNumber=%d AND fExcludedFDAKEY=1", from);
323
324 res = serv.Query(query);
325 if (!res)
326 return kFALSE;
327
328 row = res->Next();
329 if (!row || !(*row)[0])
330 {
331 cout << "ERROR - No result from query: " << query << endl;
332 return kFALSE;
333 }
334
335 TString start((*row)[0]);
336
337 delete res;
338
339 // ========== Request data of sequence ==========
340 query = Form("SELECT fSourceKEY, fProjectKEY, "
341 " fL1TriggerTableKEY, fL1TriggerTableKEY,"
342 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
343 " fTriggerDelayTableKEY, fLightConditionsKEY, fTestFlagKEY"
344 " FROM RunData"
345 " WHERE fRunTypeKEY=2 AND fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
346 " LIMIT 1", from, to);
347
348 res = serv.Query(query);
349 if (!res)
350 return kFALSE;
351
352 row = res->Next();
353 if (!row)
354 {
355 cout << "ERROR - No result from query: " << query << endl;
356 return kFALSE;
357 }
358
359 TString query1("INSERT Sequences SET");
360 query1+=Form(" fSequenceFirst=%d, fSequenceLast=%d,", from, to);
361 query1+=Form(" fProjectKEY=%s,", (*row)[0]);
362 query1+=Form(" fSourceKEY=%s,", (*row)[1]);
363 query1+=Form(" fNumEvents=%s,", nevts.Data());
364 query1+=Form(" fRunTime=%s,", secs.Data());
365 query1+=Form(" fRunStart=\"%s\",", start.Data());
366 query1+=Form(" fZenithDistanceMin=%s,", zdmin.Data());
367 query1+=Form(" fZenithDistanceMax=%s,", zdmax.Data());
368 query1+=Form(" fAzimuthMin=%s,", azmin.Data());
369 query1+=Form(" fAzimuthMax=%s,", azmax.Data());
370 query1+=Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
371 query1+=Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
372 query1+=Form(" fHvSettingsKEY=%s,", (*row)[4]);
373 query1+=Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
374 query1+=Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
375 query1+=Form(" fLightConditionsKEY=%s,", (*row)[7]);
376 query1+=Form(" fTestFlagKEY=%s, fManuallyChangedKEY=1", (*row)[8]);
377
378
379 TString query2 = Form("UPDATE RunData SET fSequenceFirst=%d WHERE"
380 " (fRunNumber BETWEEN %d AND %d) AND"
381 " (fRunTypeKEY BETWEEN 2 AND 4) AND"
382 " fSourceKEY=%d AND fHvSettingsKEY=%s AND fExcludedFDAKEY=1",
383 from, from, to, (*row)[1], (*row)[4]);
384
385 TString query3 = Form("INSERT SequenceProcessStatus SET fSequenceFirst=%d ", from);
386
387 delete res;
388
389 res = serv.Query(query1);
390 if (!res)
391 return kFALSE;
392 delete res;
393
394 res = serv.Query(query2);
395 if (!res)
396 return kFALSE;
397 delete res;
398
399 res = serv.Query(query3);
400 if (!res)
401 return kFALSE;
402 delete res;
403
404 return kTRUE;
405}
406
407Bool_t NewSequence(MSQLServer &serv, Int_t from, Int_t to, Bool_t dummy)
408{
409 cout << "Found Sequence (" << from << ", " << to << ") ... checking runs..." << flush;
410
411 if (!CheckRuns(serv, from, to, 2))
412 {
413 cout << "Warning - Found inconsistency in data-runs (" << from << ", " << to << ")" << endl;
414 //sequence is not built, but kTRUE is returned, to allow
415 //the automatic processing of the other sequences of this day
416 return kTRUE;
417 }
418 if (!CheckRuns(serv, from, to, 3))
419 {
420 cout << "Warning - Found inconsistency in ped-runs (" << from << ", " << to << ")" << endl;
421 //sequence is not built, but kTRUE is returned, to allow
422 //the automatic processing of the other sequences of this day
423 return kTRUE;
424 }
425
426 cout << "ok." << endl;
427
428
429 cout << "checking Sequence..." << endl;
430
431 if (!CheckSequence(serv, from, to, dummy))
432 {
433 cout << " inserting sequence not necessary" << endl;
434 return kTRUE;
435 }
436
437
438 if (dummy)
439 return kTRUE;
440
441 Bool_t rc = InsertSequence(serv, from, to);
442 if (!rc)
443 cout << "InsertSequence failed!" << endl;
444
445 return rc;
446}
447
448Bool_t Process(MSQLServer &serv, Int_t from, Int_t to, Bool_t dummy)
449{
450
451 TString query(Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
452 " FROM RunData"
453 " WHERE fRunNumber BETWEEN %d AND %d AND "
454 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
455 " ORDER BY fRunNumber", from, to));
456
457 TSQLResult *res = serv.Query(query);
458 if (!res)
459 return kFALSE;
460
461 TExMap map;
462
463 Int_t start=0;
464 Int_t stop=0;
465 Int_t last=0;
466 Int_t first=0;
467
468 MTime lasttime;
469
470 TSQLRow *row=0;
471
472 enum { UNKNOWN, PED=3, CAL=4, DATA=2 };
473 Char_t status = UNKNOWN;
474
475 Int_t nblocks = 0;
476
477 while ((row=res->Next()))
478 {
479 if (!(*row)[1])
480 continue;
481
482 if (start==0)
483 {
484 first = atoi((*row)[0]);
485 if (debug)
486 cout << "First Run: " << first << endl;
487 }
488
489 switch (atoi((*row)[1]))
490 {
491 case CAL: // ---------- CALIBRATION ----------
492 if (status!=CAL)
493 {
494 start = stop = atoi((*row)[0]);
495 if (!(*row)[2])
496 cout << "No time available... skipped." << endl;
497 else
498 {
499 MTime *tm = new MTime;
500 tm->SetSqlDateTime((*row)[2]);
501 map.Add((ULong_t)map.GetSize(), (Long_t)tm, (Long_t)nblocks);
502 }
503 }
504 status = CAL;
505 break;
506 default:
507 if (status==CAL)
508 {
509 MTime *tm = new MTime(lasttime);
510 map.Add((ULong_t)map.GetSize(), (Long_t)tm, (Long_t)nblocks);
511
512 stop = last;
513 nblocks++;
514 if (debug)
515 cout << "Cal Block #" << nblocks << " from " << start << " to " << last << endl;
516 }
517 status = UNKNOWN;
518 break;
519 }
520 last = atoi((*row)[0]);
521 lasttime.SetSqlDateTime((*row)[3]);
522 }
523 if (status==CAL)
524 {
525 stop = last;
526 nblocks++;
527 if (debug)
528 cout << "Cal Block #" << nblocks << " from " << start << " to " << stop << endl;
529 }
530
531 if (debug)
532 cout << "Last Run: " << last << endl;
533 delete res;
534
535 if (debug)
536 cout << "Found " << nblocks << " calibration blocks" << endl;
537
538 res = serv.Query(query);
539 if (!res)
540 return kFALSE;
541
542 Int_t n = -1;
543
544 Bool_t rc = kTRUE;
545
546 start = first;
547 while ((row=res->Next()))
548 {
549 if (!(*row)[1])
550 continue;
551
552 MTime tstart, tstop;
553 tstart.SetSqlDateTime((*row)[2]);
554 tstop.SetSqlDateTime((*row)[3]);
555
556 MTime min;
557 Int_t nmin = -1;
558 Double_t dmin = 1e35;
559
560 Long_t key, val;
561 TExMapIter nmap(&map);
562 while (nmap.Next(key, val))
563 {
564 MTime *t = (MTime*)key;
565
566 if (nmin==-1)
567 {
568 nmin = val;
569 min = *(MTime*)key;
570 dmin = fabs((Double_t)*t-(Double_t)tstart);
571 }
572
573 if (fabs((Double_t)*t-(Double_t)tstart) < dmin)
574 {
575 nmin = val;
576 dmin = fabs((Double_t)*t-(Double_t)tstart);
577 min = *t;
578 }
579 if (fabs((Double_t)*t-(Double_t)tstop) < dmin)
580 {
581 nmin = val;
582 dmin = fabs((Double_t)*t-(Double_t)tstop);
583 min = *t;
584 }
585 }
586
587 if (n!=nmin)
588 {
589 if (n!=-1)
590 {
591 if (!NewSequence(serv, start, last, dummy))
592 {
593 rc = kFALSE;
594 //continue;
595 }
596 }
597 n = nmin;
598 start = atoi((*row)[0]);
599 }
600 last = atoi((*row)[0]);
601 }
602
603 delete res;
604
605 if (n!=-1 && start!=last)
606 {
607 if (!NewSequence(serv, start, last, dummy))
608 rc = kFALSE;
609 }
610
611 if (debug)
612 cout << endl;
613
614 return rc;
615}
616
617
618
619int buildsequenceentries(TString day, Bool_t dummy=kTRUE)
620{
621 TEnv env("sql.rc");
622
623 MSQLServer serv(env);
624 if (!serv.IsConnected())
625 {
626 cout << "ERROR - Connection to database failed." << endl;
627 return 0;
628 }
629
630 cout << "buildsequences" << endl;
631 cout << "--------------" << endl;
632 cout << endl;
633 cout << "Connected to " << serv.GetName() << endl;
634 cout << "Night of sunrise at: " << day << endl;
635 cout << endl;
636
637 day += " 13:00:00";
638 const TString cond(Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\")",
639 day.Data(), day.Data()));
640
641 TString query(Form("SELECT fRunNumber, fSourceKEY, fProjectKEY, fHvSettingsKEY, fLightConditionsKEY, fDiscriminatorThresholdTableKEY, fTriggerDelayTableKEY FROM RunData WHERE %s AND fExcludedFDAKEY=1 order by fRunNumber", cond.Data()));
642
643 TSQLResult *res = serv.Query(query);
644 if (!res)
645 return 0;
646
647 TString keys[6]= { "NULL", "NULL", "NULL", "NULL", "NULL", "NULL" };
648 TString stop = "NULL";
649 TString runstart = "NULL";
650 TString runstop = "NULL";
651 Int_t count = 0;
652 TExMap blocks;
653 Int_t runbegin;
654 Int_t runend;
655
656 TSQLRow *row=0;
657 while ((row=res->Next()))
658 {
659 if (count==0)
660 {
661 for (Int_t i=1 ; i<7 ; i++)
662 keys[i-1]=(*row)[i];
663 runstart=(*row)[0];
664 }
665
666 for (Int_t i=1 ; i<7 ; i++)
667 {
668 runbegin=atoi(runstart.Data());
669 runend=atoi(runstop.Data());
670 if (i==2 && runbegin>20100 && runend<45100)
671 continue;
672
673 TString value=(*row)[i];
674 TString key=keys[i-1];
675 if (!value.CompareTo(key))
676 continue;
677
678 keys[i-1]=value;
679 //hier einfuellen
680 blocks.Add((ULong_t)blocks.GetSize(), (Long_t)runbegin, (Long_t)runend);
681 runstart=(*row)[0];
682 for (Int_t i=1 ; i<7 ; i++)
683 keys[i-1]=(*row)[i];
684 break;
685 }
686 runstop=(*row)[0];
687 count++;
688 }
689
690 //und hier einfuellen (letzter wert)
691 runbegin=atoi(runstart.Data());
692 runend=atoi(runstop.Data());
693 blocks.Add((ULong_t)blocks.GetSize(), (Long_t)runbegin, (Long_t)runend);
694
695
696 Bool_t rc = kTRUE;
697
698 Long_t key, val;
699 TExMapIter nblocks(&blocks);
700 while (nblocks.Next(key, val))
701 {
702 Int_t runstart2 = (Int_t)key;
703 Int_t runstop2 = (Int_t)val;
704 cout << endl << "datablock from " << runstart2 << " to " << runstop2 << endl;
705
706 if (!Process(serv, runstart2, runstop2, dummy))
707 rc = kFALSE;
708
709 }
710 return rc ? 1 : 0;
711}
712
713
714int buildsequenceentries()
715{
716 TEnv env("sql.rc");
717
718 MSQLServer serv(env);
719 if (!serv.IsConnected())
720 {
721 cout << "ERROR - Connection to database failed." << endl;
722 return 0;
723 }
724
725 TString query="SELECT fDate FROM SequenceBuildStatus";
726
727 TSQLResult *res = serv.Query(query);
728 if (!res)
729 return 0;
730
731 TSQLRow *row=0;
732 while ((row=res->Next()))
733 {
734 cout << "date: " << (*row)[0] << endl;
735 buildsequenceentries((*row)[0]);
736 }
737
738 return 1;
739}
Note: See TracBrowser for help on using the repository browser.