source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 6940

Last change on this file since 6940 was 6933, checked in by Daniela Dorner, 21 years ago
*** empty log message ***
File size: 17.3 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2004
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - sorts the runs by source
34// - groups the runs to sequences such that each run belongs
35// to the nearest (in time) calibration run
36//
37// apart from that the runs in one sequence must have the same
38// hv-settings, trigger tables, light conditions, DT tables,
39// trigger delay tables and testflag
40//
41/////////////////////////////////////////////////////////////////////////////
42#include <iostream>
43#include <iomanip>
44#include <fstream>
45
46#include <MSQLServer.h>
47#include <TSQLRow.h>
48#include <TSQLResult.h>
49
50#include <TEnv.h>
51#include <TMath.h>
52#include <TExMap.h>
53#include <TArrayI.h>
54#include <TRegexp.h>
55#include <TSystem.h>
56
57#include <MTime.h>
58#include <MDirIter.h>
59
60using namespace std;
61
62int debug = 0;
63
64Bool_t DeleteSequences(MSQLServer &serv, TString cond)
65{
66 TString query(Form("SELECT fSequenceFirst FROM MyMagic.Sequences"
67 " WHERE %s AND fManuallyChangedKEY=1", cond.Data()));
68
69 TSQLResult *res = serv.Query(query);
70 if (!res)
71 return kFALSE;
72
73 TSQLRow *row=0;
74 while ((row=res->Next()))
75 {
76 query = Form("DELETE FROM MyMagic.Calibration WHERE fSequenceFirst=%s", (*row)[0]);
77
78 TSQLResult *res = serv.Query(query);
79 if (!res)
80 return kFALSE;
81 delete res;
82
83 query = Form("DELETE FROM MyMagic.SequenceProcessStatus WHERE fSequenceFirst=%s", (*row)[0]);
84
85 res = serv.Query(query);
86 if (!res)
87 return kFALSE;
88 delete res;
89
90 Int_t sequno=atoi((*row)[0]);
91 TString fname(Form("/mnt/stk01/sequences/%04d/sequence%08d.txt", sequno/10000, sequno));
92 gSystem->Unlink(fname);
93
94 query = Form("UPDATE MyMagic.RunData SET fSequenceFirst=0 WHERE fSequenceFirst=%s", (*row)[0]);
95
96 res = serv.Query(query);
97 if (!res)
98 return kFALSE;
99 delete res;
100
101 }
102 delete res;
103
104 query = Form("DELETE FROM MyMagic.Sequences WHERE %s AND fManuallyChangedKEY=1", cond.Data());
105 res = serv.Query(query);
106 if (!res)
107 return kFALSE;
108
109 delete res;
110
111 return kTRUE;
112}
113
114Int_t CheckOverlap(MSQLServer &serv, Int_t from, Int_t to)
115{
116 TString query(Form("SELECT fSequenceFirst FROM MyMagic.Sequences WHERE"
117 " (%d BETWEEN fSequenceFirst AND fSequenceLast) OR "
118 " (%d BETWEEN fSequenceFirst AND fSequenceLast)", from, to));
119
120 TSQLResult *res = serv.Query(query);
121 if (!res)
122 return -1;
123
124 TSQLRow *row = res->Next();
125
126 Int_t rc = row ? kFALSE : kTRUE;
127 if (rc==kFALSE)
128 cout << "Sorry - the sequence from " << from << " to " << to << " overlaps with sequence #" << (*row)[0] << endl;
129
130 delete res;
131
132 return rc;
133}
134
135Int_t DoCheck(TSQLResult &res, Int_t from, Int_t to)
136{
137 TSQLRow *row=0;
138
139 TArrayI data(9);
140
141 Int_t n = 0;
142
143 while ((row=res.Next()))
144 {
145 n++;
146
147 if (data[0]==0)
148 {
149 for (int i=0; i<data.GetSize(); i++)
150 data[i] = atoi((*row)[i]);
151 continue;
152 }
153
154 for (int i=1; i<data.GetSize(); i++)
155 {
156 if (data[i] != atoi((*row)[i]))
157 return i+1;
158 }
159 }
160 return n==0 ? 0 : -1;
161}
162
163Bool_t CheckSequence(MSQLServer &serv, Int_t from, Int_t to, Int_t src, Int_t type)
164{
165 TString query("SELECT fRunNumber, fL1TriggerTableKEY, fL2TriggerTableKEY,"
166 " fProjectKEY, fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
167 " fTriggerDelayTableKEY, fLightConditionsKEY, fTestFlagKEY"
168 " FROM MyMagic.RunData");
169 query += Form(" WHERE fRunTypeKEY=%d AND fSourceKEY=%d AND fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
170 " ORDER BY fRunNumber", type, src, from, to);
171
172 TSQLResult *res = serv.Query(query);
173 if (!res)
174 return kFALSE;
175
176 Int_t rc = DoCheck(*res, from, to);
177 delete res;
178
179 switch (rc)
180 {
181 case 0: cout << "ERROR - No runs found for check!" << endl; break;
182 case 1: cout << "ERROR - fRunNumber doesn't match!" << endl; break;
183 case 2: cout << "ERROR - fL1TriggerTableKEY doesn't match!" << endl; break;
184 case 3: cout << "ERROR - fL2TriggerTableKEY doesn't match!" << endl; break;
185 case 4: cout << "ERROR - fProjectKEY doesn't match!" << endl; break;
186 case 5: cout << "ERROR - fHvSettingsKEY doesn't match!" << endl; break;
187 case 6: cout << "ERROR - fDiscriminatorThresholdTableKEY doesn't match!" << endl; break;
188 case 7: cout << "ERROR - fTriggerDelayTableKEY doesn't match!" << endl; break;
189 case 8: cout << "ERROR - fLightConditionsKEY doesn't match!" << endl; break;
190 case 9: cout << "ERROR - fTestFlagKEY doesn't match!" << endl; break;
191 }
192
193 return rc<0;
194}
195
196Bool_t InsertSequence(MSQLServer &serv, Int_t from, Int_t to, Int_t src, Bool_t dummy)
197{
198 Int_t rc = dummy ? kTRUE : CheckOverlap(serv, from, to);
199 if (rc<=0)
200 return kFALSE;
201
202 // ========== Request number of events ==========
203 TString query("SELECT SUM(fNumEvents), "
204 " SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
205 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
206 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ");
207 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
208 query += " MIN(fAzimuth), MAX(fAzimuth) ";
209 query += Form(" FROM MyMagic.RunData"
210 " WHERE fRunTypeKEY=2 AND fSourceKEY=%d AND (fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
211 src, from, to);
212
213 TSQLResult *res = serv.Query(query);
214 if (!res)
215 return kFALSE;
216
217 TSQLRow *row = res->Next();
218 if (!row || !(*row)[0])
219 {
220 cout << "ERROR - No result from query: " << query << endl;
221 return kFALSE;
222 }
223
224 TString nevts = (*row)[0];
225 TString secs = (*row)[1];
226 TString zdmin = (*row)[2];
227 TString zdmax = (*row)[3];
228 TString azmin = (*row)[4];
229 TString azmax = (*row)[5];
230
231 delete res;
232
233 // ========== Request start time of sequence ==========
234 query = Form("SELECT fRunStart FROM MyMagic.RunData WHERE fRunNumber=%d AND fSourceKEY=%d AND fExcludedFDAKEY=1", from, src);
235
236 res = serv.Query(query);
237 if (!res)
238 return kFALSE;
239
240 row = res->Next();
241 if (!row || !(*row)[0])
242 {
243 cout << "ERROR - No result from query: " << query << endl;
244 return kFALSE;
245 }
246
247 TString start((*row)[0]);
248
249 delete res;
250
251 // ========== Request data of sequence ==========
252 query = Form("SELECT fProjectKEY, fL1TriggerTableKEY, fL1TriggerTableKEY,"
253 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
254 " fTriggerDelayTableKEY, fLightConditionsKEY, fTestFlagKEY"
255 " FROM MyMagic.RunData"
256 " WHERE fRunTypeKEY=2 AND fSourceKEY=%d AND fExcludedFDAKEY=1 AND (fRunNumber BETWEEN %d AND %d)"
257 " LIMIT 1", src, from, to);
258
259 res = serv.Query(query);
260 if (!res)
261 return kFALSE;
262
263 row = res->Next();
264 if (!row)
265 {
266 cout << "ERROR - No result from query: " << query << endl;
267 return kFALSE;
268 }
269
270 TString query1 = Form("INSERT MyMagic.Sequences SET"
271 " fSequenceFirst=%d,"
272 " fSequenceLast=%d,"
273 " fProjectKEY=%s,"
274 " fSourceKEY=%d,"
275 " fNumEvents=%s,"
276 " fRunTime=%s,"
277 " fRunStart=\"%s\","
278 " fZenithDistanceMin=%s,"
279 " fZenithDistanceMax=%s,"
280 " fAzimuthMin=%s,"
281 " fAzimuthMax=%s,"
282 " fL1TriggerTableKEY=%s,"
283 " fL2TriggerTableKEY=%s,"
284 " fHvSettingsKEY=%s,"
285 " fDiscriminatorThresholdTableKEY=%s,"
286 " fTriggerDelayTableKEY=%s,"
287 " fLightConditionsKEY=%s,"
288 " fTestFlagKEY=%s,"
289 " fManuallyChangedKEY=1",
290 from, to, (*row)[0], src, nevts.Data(),
291 secs.Data(), start.Data(), zdmin.Data(),
292 zdmax.Data(), azmin.Data(), azmax.Data(),
293 (*row)[1], (*row)[2], (*row)[3],
294 (*row)[4], (*row)[5], (*row)[6],
295 (*row)[7]);
296
297 TString query2 = Form("UPDATE MyMagic.RunData SET fSequenceFirst=%d WHERE"
298 " (fRunNumber BETWEEN %d AND %d) AND"
299 " (fRunTypeKEY BETWEEN 2 AND 4) AND"
300 " fSourceKEY=%d AND fHvSettingsKEY=%s AND fExcludedFDAKEY=1",
301 from, from, to, src, (*row)[3]);
302
303 TString query3 = Form("INSERT MyMagic.SequenceProcessStatus SET fSequenceFirst=%d ",
304 from);
305 delete res;
306
307 if (dummy)
308 return kTRUE;
309
310 res = serv.Query(query1);
311 if (!res)
312 return kFALSE;
313 delete res;
314
315 res = serv.Query(query2);
316 if (!res)
317 return kFALSE;
318 delete res;
319
320 res = serv.Query(query3);
321 if (!res)
322 return kFALSE;
323 delete res;
324
325 return kTRUE;
326}
327
328Bool_t NewSequence(MSQLServer &serv, Int_t from, Int_t to, Int_t src, Bool_t dummy)
329{
330 cout << "Found Sequence (" << from << ", " << to << ") ... checking..." << flush;
331
332 if (!CheckSequence(serv, from, to, src, 2))
333 {
334 cout << "Warning - Found inconsistency in data-runs (" << from << ", " << to << ")" << endl;
335 //sequence is not built, but kTRUE is returned, to allow
336 //the automatic processing of the other sequences of this day
337 return kTRUE;
338 }
339 if (!CheckSequence(serv, from, to, src, 3))
340 {
341 cout << "Warning - Found inconsistency in ped-runs (" << from << ", " << to << ")" << endl;
342 //sequence is not built, but kTRUE is returned, to allow
343 //the automatic processing of the other sequences of this day
344 return kTRUE;
345 }
346
347 cout << "ok." << endl;
348
349 Bool_t rc = InsertSequence(serv, from, to, src, dummy);
350 if (!rc)
351 cout << "InsertSequence failed!" << endl;
352
353 return rc;
354}
355
356Bool_t GetSources(MSQLServer &serv, TString cond, TArrayI &srcs)
357{
358 TString query(Form("SELECT fSourceKEY"
359 " FROM MyMagic.RunData"
360 " WHERE %s AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) GROUP BY fSourceKEY",
361 cond.Data())
362 ); //DATE(fStartTime)=\"2004-05-19\"");
363
364 TSQLResult *res = serv.Query(query);
365 if (!res)
366 return kFALSE;
367
368 srcs.Set(res->GetRowCount());
369
370 cout << "Found " << srcs.GetSize() << " sources." << endl << endl;
371
372 TSQLRow *row=0;
373 Int_t i=0;
374 while ((row=res->Next()))
375 srcs[i++] = atoi((*row)[0]);
376
377 delete res;
378 return kTRUE;
379}
380
381Bool_t Process(MSQLServer &serv, TString cond, Int_t src, Bool_t dummy)
382{
383 if (debug)
384 cout << "Processing Source: " << src << endl;
385
386 TString query(Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
387 " FROM MyMagic.RunData"
388 " WHERE %s AND fSourceKEY=%d AND fExcludedFDAKEY=1 AND"
389 " (fRunTypeKEY BETWEEN 2 AND 4)"
390 " ORDER BY fRunNumber", cond.Data(), src)
391 ); //DATE(fStartTime)=\"2004-05-19\"");
392
393 TSQLResult *res = serv.Query(query);
394 if (!res)
395 return kFALSE;
396
397 TExMap map;
398
399 Int_t start=0;
400 Int_t stop=0;
401 Int_t last=0;
402 Int_t first=0;
403
404 MTime lasttime;
405
406 TSQLRow *row=0;
407
408 enum { UNKNOWN, PED=3, CAL=4, DATA=2 };
409 Char_t status = UNKNOWN;
410
411 Int_t nblocks = 0;
412
413 while ((row=res->Next()))
414 {
415 if (!(*row)[1])
416 continue;
417
418 if (start==0)
419 {
420 first = atoi((*row)[0]);
421 if (debug)
422 cout << "First Run: " << first << endl;
423 }
424
425 switch (atoi((*row)[1]))
426 {
427 case CAL: // ---------- CALIBRATION ----------
428 if (status!=CAL)
429 {
430 start = stop = atoi((*row)[0]);
431 if (!(*row)[2])
432 cout << "No time available... skipped." << endl;
433 else
434 {
435 MTime *tm = new MTime;
436 tm->SetSqlDateTime((*row)[2]);
437 map.Add((ULong_t)map.GetSize(), (Long_t)nblocks, (Long_t)tm);
438 }
439 }
440 status = CAL;
441 break;
442 default:
443 if (status==CAL)
444 {
445 MTime *tm = new MTime(lasttime);
446 map.Add((ULong_t)map.GetSize(), (Long_t)nblocks, (Long_t)tm);
447
448 stop = last;
449 nblocks++;
450 if (debug)
451 cout << "Cal Block #" << nblocks << " from " << start << " to " << last << endl;
452 }
453 status = UNKNOWN;
454 break;
455 }
456 last = atoi((*row)[0]);
457 lasttime.SetSqlDateTime((*row)[3]);
458 }
459 if (status==CAL)
460 {
461 stop = last;
462 nblocks++;
463 if (debug)
464 cout << "Cal Block #" << nblocks << " from " << start << " to " << stop << endl;
465 }
466
467 if (debug)
468 cout << "Last Run: " << last << endl;
469 delete res;
470
471 if (debug)
472 cout << "Found " << nblocks << " calibration blocks" << endl;
473
474 res = serv.Query(query);
475 if (!res)
476 return kFALSE;
477
478 Int_t n = -1;
479
480 Bool_t rc = kTRUE;
481
482 start = first;
483 while ((row=res->Next()))
484 {
485 if (!(*row)[1])
486 continue;
487
488 MTime tstart, tstop;
489 tstart.SetSqlDateTime((*row)[2]);
490 tstop.SetSqlDateTime((*row)[3]);
491
492 MTime min;
493 Int_t nmin = -1;
494 Double_t dmin = 1e35;
495
496 Long_t key, val;
497 TExMapIter nmap(&map);
498 while (nmap.Next(key, val))
499 {
500 MTime *t = (MTime*)val;
501
502 if (nmin==-1)
503 {
504 nmin = key;
505 min = *(MTime*)val;
506 dmin = fabs((Double_t)*t-(Double_t)tstart);
507 }
508
509 if (fabs((Double_t)*t-(Double_t)tstart) < dmin)
510 {
511 nmin = key;
512 dmin = fabs((Double_t)*t-(Double_t)tstart);
513 min = *t;
514 }
515 if (fabs((Double_t)*t-(Double_t)tstop) < dmin)
516 {
517 nmin = key;
518 dmin = fabs((Double_t)*t-(Double_t)tstop);
519 min = *t;
520 }
521 }
522
523 if (n!=nmin)
524 {
525 if (n!=-1)
526 {
527 if (!NewSequence(serv, start, last, src, dummy))
528 {
529 rc = kFALSE;
530 //continue;
531 }
532 }
533 n = nmin;
534 start = atoi((*row)[0]);
535 }
536 last = atoi((*row)[0]);
537 }
538
539 delete res;
540
541 if (n!=-1 && start!=last)
542 {
543 if (!NewSequence(serv, start, last, src, dummy))
544 rc = kFALSE;
545 }
546
547 if (debug)
548 cout << endl;
549
550 return rc;
551}
552
553// This tool will work from Period017 (2004_05_17) on...
554int buildsequenceentries(TString day, Bool_t dummy=kTRUE)
555{
556 TEnv env("sql.rc");
557
558 MSQLServer serv(env);
559 if (!serv.IsConnected())
560 {
561 cout << "ERROR - Connection to database failed." << endl;
562 return 0;
563 }
564
565 cout << "buildsequences" << endl;
566 cout << "--------------" << endl;
567 cout << endl;
568 cout << "Connected to " << serv.GetName() << endl;
569 cout << "Night of sunrise at: " << day << endl;
570 cout << endl;
571
572 day += " 13:00:00";
573 const TString cond(Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\")",
574 day.Data(), day.Data()));
575
576 if (!dummy && !DeleteSequences(serv, cond))
577 return 0;
578
579 Bool_t rc = kTRUE;
580
581 // get all sources for this day
582 TArrayI src;
583 GetSources(serv, cond, src);
584 // find and build the sequences for the day for each source
585 for (int i=0; i<src.GetSize(); i++)
586 if (!Process(serv, cond, src[i], dummy))
587 rc = kFALSE;
588
589 return rc ? 1 : 0;
590}
Note: See TracBrowser for help on using the repository browser.