source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 7183

Last change on this file since 7183 was 7183, checked in by Daniela Dorner, 19 years ago
*** empty log message ***
File size: 17.9 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2004
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - sorts the runs by source
34// - groups the runs to sequences such that each run belongs
35// to the nearest (in time) calibration run
36//
37// apart from that the runs in one sequence must have the same
38// hv-settings, trigger tables, light conditions, DT tables,
39// trigger delay tables and testflag
40//
41/////////////////////////////////////////////////////////////////////////////
42#include <iostream>
43#include <iomanip>
44#include <fstream>
45
46#include <MSQLServer.h>
47#include <TSQLRow.h>
48#include <TSQLResult.h>
49
50#include <TEnv.h>
51#include <TMath.h>
52#include <TExMap.h>
53#include <TArrayI.h>
54#include <TRegexp.h>
55#include <TSystem.h>
56
57#include <MTime.h>
58#include <MDirIter.h>
59
60using namespace std;
61
62int debug = 0;
63
64Bool_t DeleteSequences(MSQLServer &serv, TString cond)
65{
66 TString query(Form("SELECT fSequenceFirst FROM Sequences"
67 " WHERE %s AND fManuallyChangedKEY=1", cond.Data()));
68
69 TSQLResult *res = serv.Query(query);
70 if (!res)
71 return kFALSE;
72
73 TSQLRow *row=0;
74 while ((row=res->Next()))
75 {
76 query = Form("DELETE FROM Calibration WHERE fSequenceFirst=%s", (*row)[0]);
77
78 TSQLResult *res = serv.Query(query);
79 if (!res)
80 return kFALSE;
81 delete res;
82
83 query = Form("DELETE FROM Star WHERE fSequenceFirst=%s", (*row)[0]);
84
85 res = serv.Query(query);
86 if (!res)
87 return kFALSE;
88 delete res;
89
90 query = Form("DELETE FROM SequenceProcessStatus WHERE fSequenceFirst=%s", (*row)[0]);
91
92 res = serv.Query(query);
93 if (!res)
94 return kFALSE;
95 delete res;
96
97 Int_t sequno=atoi((*row)[0]);
98 TString fname(Form("/mnt/stk01/sequences/%04d/sequence%08d.txt", sequno/10000, sequno));
99 gSystem->Unlink(fname);
100
101 query = Form("UPDATE RunData SET fSequenceFirst=0 WHERE fSequenceFirst=%s", (*row)[0]);
102
103 res = serv.Query(query);
104 if (!res)
105 return kFALSE;
106 delete res;
107
108 }
109 delete res;
110
111 query = Form("DELETE FROM Sequences WHERE %s AND fManuallyChangedKEY=1", cond.Data());
112 res = serv.Query(query);
113 if (!res)
114 return kFALSE;
115
116 delete res;
117
118 return kTRUE;
119}
120
121Int_t CheckOverlap(MSQLServer &serv, Int_t from, Int_t to)
122{
123 TString query(Form("SELECT fSequenceFirst FROM Sequences WHERE"
124 " (%d BETWEEN fSequenceFirst AND fSequenceLast) OR "
125 " (%d BETWEEN fSequenceFirst AND fSequenceLast)", from, to));
126
127 TSQLResult *res = serv.Query(query);
128 if (!res)
129 return -1;
130
131 TSQLRow *row = res->Next();
132
133 Int_t rc = row ? kFALSE : kTRUE;
134 if (rc==kFALSE)
135 cout << "Sorry - the sequence from " << from << " to " << to << " overlaps with sequence #" << (*row)[0] << endl;
136
137 delete res;
138
139 return rc;
140}
141
142Int_t DoCheck(TSQLResult &res, Int_t from, Int_t to)
143{
144 TSQLRow *row=0;
145
146 TArrayI data(9);
147
148 Int_t n = 0;
149
150 while ((row=res.Next()))
151 {
152 n++;
153
154 if (data[0]==0)
155 {
156 for (int i=0; i<data.GetSize(); i++)
157 data[i] = atoi((*row)[i]);
158 continue;
159 }
160
161 for (int i=1; i<data.GetSize(); i++)
162 {
163 if (data[i] != atoi((*row)[i]))
164 return i+1;
165 }
166 }
167 return n==0 ? 0 : -1;
168}
169
170Bool_t CheckSequence(MSQLServer &serv, Int_t from, Int_t to, Int_t src, Int_t type)
171{
172 TString query("SELECT fRunNumber, fL1TriggerTableKEY, fL2TriggerTableKEY,"
173 " fProjectKEY, fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
174 " fTriggerDelayTableKEY, fLightConditionsKEY, fTestFlagKEY"
175 " FROM RunData");
176 query += Form(" WHERE fRunTypeKEY=%d AND fSourceKEY=%d AND fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
177 " ORDER BY fRunNumber", type, src, from, to);
178
179 TSQLResult *res = serv.Query(query);
180 if (!res)
181 return kFALSE;
182
183 Int_t rc = DoCheck(*res, from, to);
184 delete res;
185
186 switch (rc)
187 {
188 case 0: cout << "ERROR - No runs found for check!" << endl; break;
189 case 1: cout << "ERROR - fRunNumber doesn't match!" << endl; break;
190 case 2: cout << "ERROR - fL1TriggerTableKEY doesn't match!" << endl; break;
191 case 3: cout << "ERROR - fL2TriggerTableKEY doesn't match!" << endl; break;
192 case 4: cout << "ERROR - fProjectKEY doesn't match!" << endl; break;
193 case 5: cout << "ERROR - fHvSettingsKEY doesn't match!" << endl; break;
194 case 6: cout << "ERROR - fDiscriminatorThresholdTableKEY doesn't match!" << endl; break;
195 case 7: cout << "ERROR - fTriggerDelayTableKEY doesn't match!" << endl; break;
196 case 8: cout << "ERROR - fLightConditionsKEY doesn't match!" << endl; break;
197 case 9: cout << "ERROR - fTestFlagKEY doesn't match!" << endl; break;
198 }
199
200 return rc<0;
201}
202
203Bool_t InsertSequence(MSQLServer &serv, Int_t from, Int_t to, Int_t src, Bool_t dummy)
204{
205 Int_t rc = dummy ? kTRUE : CheckOverlap(serv, from, to);
206 if (rc<=0)
207 return kFALSE;
208
209 // ========== Request number of events ==========
210 TString query("SELECT SUM(fNumEvents), "
211 " SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
212 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
213 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ");
214 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
215 query += " MIN(fAzimuth), MAX(fAzimuth) ";
216 query += Form(" FROM RunData"
217 " WHERE fRunTypeKEY=2 AND fSourceKEY=%d AND (fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
218 src, from, to);
219
220 TSQLResult *res = serv.Query(query);
221 if (!res)
222 return kFALSE;
223
224 TSQLRow *row = res->Next();
225 if (!row || !(*row)[0])
226 {
227 cout << "ERROR - No result from query: " << query << endl;
228 return kFALSE;
229 }
230
231 TString nevts = (*row)[0];
232 TString secs = (*row)[1];
233 TString zdmin = (*row)[2];
234 TString zdmax = (*row)[3];
235 TString azmin = (*row)[4];
236 TString azmax = (*row)[5];
237
238 delete res;
239
240 // ========== Request start time of sequence ==========
241 query = Form("SELECT fRunStart FROM RunData WHERE fRunNumber=%d AND fSourceKEY=%d AND fExcludedFDAKEY=1", from, src);
242
243 res = serv.Query(query);
244 if (!res)
245 return kFALSE;
246
247 row = res->Next();
248 if (!row || !(*row)[0])
249 {
250 cout << "ERROR - No result from query: " << query << endl;
251 return kFALSE;
252 }
253
254 TString start((*row)[0]);
255
256 delete res;
257
258 // ========== Request data of sequence ==========
259 query = Form("SELECT fProjectKEY, fL1TriggerTableKEY, fL1TriggerTableKEY,"
260 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
261 " fTriggerDelayTableKEY, fLightConditionsKEY, fTestFlagKEY"
262 " FROM RunData"
263 " WHERE fRunTypeKEY=2 AND fSourceKEY=%d AND fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
264 " LIMIT 1", src, from, to);
265
266 res = serv.Query(query);
267 if (!res)
268 return kFALSE;
269
270 row = res->Next();
271 if (!row)
272 {
273 cout << "ERROR - No result from query: " << query << endl;
274 return kFALSE;
275 }
276
277 TString query1 = Form("INSERT Sequences SET"
278 " fSequenceFirst=%d,"
279 " fSequenceLast=%d,"
280 " fProjectKEY=%s,"
281 " fSourceKEY=%d,"
282 " fNumEvents=%s,"
283 " fRunTime=%s,"
284 " fRunStart=\"%s\","
285 " fZenithDistanceMin=%s,"
286 " fZenithDistanceMax=%s,"
287 " fAzimuthMin=%s,"
288 " fAzimuthMax=%s,"
289 " fL1TriggerTableKEY=%s,"
290 " fL2TriggerTableKEY=%s,"
291 " fHvSettingsKEY=%s,"
292 " fDiscriminatorThresholdTableKEY=%s,"
293 " fTriggerDelayTableKEY=%s,"
294 " fLightConditionsKEY=%s,"
295 " fTestFlagKEY=%s,"
296 " fManuallyChangedKEY=1",
297 from, to, (*row)[0], src, nevts.Data(),
298 secs.Data(), start.Data(), zdmin.Data(),
299 zdmax.Data(), azmin.Data(), azmax.Data(),
300 (*row)[1], (*row)[2], (*row)[3],
301 (*row)[4], (*row)[5], (*row)[6],
302 (*row)[7]);
303
304 TString query2 = Form("UPDATE RunData SET fSequenceFirst=%d WHERE"
305 " (fRunNumber BETWEEN %d AND %d) AND"
306 " (fRunTypeKEY BETWEEN 2 AND 4) AND"
307 " fSourceKEY=%d AND fHvSettingsKEY=%s AND fExcludedFDAKEY=1",
308 from, from, to, src, (*row)[3]);
309
310 TString query3 = Form("INSERT SequenceProcessStatus SET fSequenceFirst=%d ",
311 from);
312 delete res;
313
314 if (dummy)
315 return kTRUE;
316
317 res = serv.Query(query1);
318 if (!res)
319 return kFALSE;
320 delete res;
321
322 res = serv.Query(query2);
323 if (!res)
324 return kFALSE;
325 delete res;
326
327 res = serv.Query(query3);
328 if (!res)
329 return kFALSE;
330 delete res;
331
332 return kTRUE;
333}
334
335Bool_t NewSequence(MSQLServer &serv, Int_t from, Int_t to, Int_t src, Bool_t dummy)
336{
337 cout << "Found Sequence (" << from << ", " << to << ") ... checking runs..." << flush;
338
339 if (!CheckSequence(serv, from, to, src, 2))
340 {
341 cout << "Warning - Found inconsistency in data-runs (" << from << ", " << to << ")" << endl;
342 //sequence is not built, but kTRUE is returned, to allow
343 //the automatic processing of the other sequences of this day
344 return kTRUE;
345 }
346 if (!CheckSequence(serv, from, to, src, 3))
347 {
348 cout << "Warning - Found inconsistency in ped-runs (" << from << ", " << to << ")" << endl;
349 //sequence is not built, but kTRUE is returned, to allow
350 //the automatic processing of the other sequences of this day
351 return kTRUE;
352 }
353
354 cout << "ok." << endl;
355
356 Bool_t rc = InsertSequence(serv, from, to, src, dummy);
357 if (!rc)
358 cout << "InsertSequence failed!" << endl;
359
360 return rc;
361}
362
363Bool_t GetSources(MSQLServer &serv, TString cond, TArrayI &srcs)
364{
365 TString query(Form("SELECT fSourceKEY"
366 " FROM RunData"
367 " WHERE %s AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) GROUP BY fSourceKEY",
368 cond.Data())
369 ); //DATE(fStartTime)=\"2004-05-19\"");
370
371 TSQLResult *res = serv.Query(query);
372 if (!res)
373 return kFALSE;
374
375 srcs.Set(res->GetRowCount());
376
377 cout << "Found " << srcs.GetSize() << " sources." << endl << endl;
378
379 TSQLRow *row=0;
380 Int_t i=0;
381 while ((row=res->Next()))
382 srcs[i++] = atoi((*row)[0]);
383
384 delete res;
385 return kTRUE;
386}
387
388Bool_t Process(MSQLServer &serv, TString cond, Int_t src, Bool_t dummy)
389{
390 if (debug)
391 cout << "Processing Source: " << src << endl;
392
393 TString query(Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
394 " FROM RunData"
395 " WHERE %s AND fSourceKEY=%d AND fExcludedFDAKEY=1 AND"
396 " (fRunTypeKEY BETWEEN 2 AND 4)"
397 " ORDER BY fRunNumber", cond.Data(), src)
398 ); //DATE(fStartTime)=\"2004-05-19\"");
399
400 TSQLResult *res = serv.Query(query);
401 if (!res)
402 return kFALSE;
403
404 TExMap map;
405
406 Int_t start=0;
407 Int_t stop=0;
408 Int_t last=0;
409 Int_t first=0;
410
411 MTime lasttime;
412
413 TSQLRow *row=0;
414
415 enum { UNKNOWN, PED=3, CAL=4, DATA=2 };
416 Char_t status = UNKNOWN;
417
418 Int_t nblocks = 0;
419
420 while ((row=res->Next()))
421 {
422 if (!(*row)[1])
423 continue;
424
425 if (start==0)
426 {
427 first = atoi((*row)[0]);
428 if (debug)
429 cout << "First Run: " << first << endl;
430 }
431
432 switch (atoi((*row)[1]))
433 {
434 case CAL: // ---------- CALIBRATION ----------
435 if (status!=CAL)
436 {
437 start = stop = atoi((*row)[0]);
438 if (!(*row)[2])
439 cout << "No time available... skipped." << endl;
440 else
441 {
442 MTime *tm = new MTime;
443 tm->SetSqlDateTime((*row)[2]);
444 map.Add((ULong_t)map.GetSize(), (Long_t)tm, (Long_t)nblocks);
445 }
446 }
447 status = CAL;
448 break;
449 default:
450 if (status==CAL)
451 {
452 MTime *tm = new MTime(lasttime);
453 map.Add((ULong_t)map.GetSize(), (Long_t)tm, (Long_t)nblocks);
454
455 stop = last;
456 nblocks++;
457 if (debug)
458 cout << "Cal Block #" << nblocks << " from " << start << " to " << last << endl;
459 }
460 status = UNKNOWN;
461 break;
462 }
463 last = atoi((*row)[0]);
464 lasttime.SetSqlDateTime((*row)[3]);
465 }
466 if (status==CAL)
467 {
468 stop = last;
469 nblocks++;
470 if (debug)
471 cout << "Cal Block #" << nblocks << " from " << start << " to " << stop << endl;
472 }
473
474 if (debug)
475 cout << "Last Run: " << last << endl;
476 delete res;
477
478 if (debug)
479 cout << "Found " << nblocks << " calibration blocks" << endl;
480
481 res = serv.Query(query);
482 if (!res)
483 return kFALSE;
484
485 Int_t n = -1;
486
487 Bool_t rc = kTRUE;
488
489 start = first;
490 while ((row=res->Next()))
491 {
492 if (!(*row)[1])
493 continue;
494
495 MTime tstart, tstop;
496 tstart.SetSqlDateTime((*row)[2]);
497 tstop.SetSqlDateTime((*row)[3]);
498
499 MTime min;
500 Int_t nmin = -1;
501 Double_t dmin = 1e35;
502
503 Long_t key, val;
504 TExMapIter nmap(&map);
505 while (nmap.Next(key, val))
506 {
507 MTime *t = (MTime*)key;
508
509 if (nmin==-1)
510 {
511 nmin = val;
512 min = *(MTime*)key;
513 dmin = fabs((Double_t)*t-(Double_t)tstart);
514 }
515
516 if (fabs((Double_t)*t-(Double_t)tstart) < dmin)
517 {
518 nmin = val;
519 dmin = fabs((Double_t)*t-(Double_t)tstart);
520 min = *t;
521 }
522 if (fabs((Double_t)*t-(Double_t)tstop) < dmin)
523 {
524 nmin = val;
525 dmin = fabs((Double_t)*t-(Double_t)tstop);
526 min = *t;
527 }
528 }
529
530 if (n!=nmin)
531 {
532 if (n!=-1)
533 {
534 if (!NewSequence(serv, start, last, src, dummy))
535 {
536 rc = kFALSE;
537 //continue;
538 }
539 }
540 n = nmin;
541 start = atoi((*row)[0]);
542 }
543 last = atoi((*row)[0]);
544 }
545
546 delete res;
547
548 if (n!=-1 && start!=last)
549 {
550 if (!NewSequence(serv, start, last, src, dummy))
551 rc = kFALSE;
552 }
553
554 if (debug)
555 cout << endl;
556
557 return rc;
558}
559
560// This tool will work from Period017 (2004_05_17) on...
561int buildsequenceentries(TString day, Bool_t dummy=kTRUE)
562{
563 TEnv env("sql.rc");
564
565 MSQLServer serv(env);
566 if (!serv.IsConnected())
567 {
568 cout << "ERROR - Connection to database failed." << endl;
569 return 0;
570 }
571
572 cout << "buildsequences" << endl;
573 cout << "--------------" << endl;
574 cout << endl;
575 cout << "Connected to " << serv.GetName() << endl;
576 cout << "Night of sunrise at: " << day << endl;
577 cout << endl;
578
579 day += " 13:00:00";
580 const TString cond(Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\")",
581 day.Data(), day.Data()));
582
583 if (!dummy && !DeleteSequences(serv, cond))
584 return 0;
585
586 Bool_t rc = kTRUE;
587
588 // get all sources for this day
589 TArrayI src;
590 GetSources(serv, cond, src);
591 // find and build the sequences for the day for each source
592 for (int i=0; i<src.GetSize(); i++)
593 if (!Process(serv, cond, src[i], dummy))
594 rc = kFALSE;
595
596 return rc ? 1 : 0;
597}
598
599int buildsequenceentries()
600{
601 TEnv env("sql.rc");
602
603 MSQLServer serv(env);
604 if (!serv.IsConnected())
605 {
606 cout << "ERROR - Connection to database failed." << endl;
607 return 0;
608 }
609
610 TString query="SELECT fDate FROM SequenceBuildStatus";
611
612 cout << "Q: " << query << endl;
613
614 TSQLResult *res = serv.Query(query);
615 if (!res)
616 return 0;
617
618 TSQLRow *row=0;
619 while ((row=res->Next()))
620 {
621 cout << "date: " << (*row)[0] << endl;
622 buildsequenceentries((*row)[0]);
623 }
624
625 return 1;
626}
Note: See TracBrowser for help on using the repository browser.