source: trunk/Mars/fact/processing/buildseqentries.C@ 19267

Last change on this file since 19267 was 18308, checked in by Daniela Dorner, 6 years ago
bugfix (start and stop in regular expressions of sequences.rc were ignored)
File size: 44.5 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82
83#include <map>
84#include <utility>
85
86#include <errno.h>
87
88#include <TSQLRow.h>
89#include <TSQLResult.h>
90
91#include <TEnv.h>
92#include <TMap.h>
93#include <TMath.h>
94#include <TExMap.h>
95#include <TArrayI.h>
96#include <TArrayL.h>
97#include <TArrayD.h>
98#include <TPRegexp.h>
99#include <TSystem.h>
100#include <TObjString.h>
101#include <TObjArray.h>
102
103#include "MTime.h"
104#include "MSequence.h"
105
106#include "MSQLMagic.h"
107
108using namespace std;
109
110 /*static*/ ULong_t GetId(const TString str)
111 {
112 const Ssiz_t dot = str.First('.');
113
114 const UInt_t run = str.Atoi();
115 const UInt_t sub = dot<0 ? 0 : atoi(str.Data()+dot+1);
116
117 return ULong_t(run)*1000+sub;
118 }
119
120
121class Rule : public TObject
122{
123private:
124 TPRegexp fRegexp;
125
126 ULong_t fMin;
127 ULong_t fMax;
128
129public:
130 Rule(TObjArray &arr) :
131 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
132 fMin(0), fMax((ULong_t)-1)
133 {
134 if (arr.GetEntries()>1)
135 fMin = GetId(arr[1]->GetName());
136 if (arr.GetEntries()>2)
137 fMax = GetId(arr[2]->GetName());
138 cout << "----> " << fMin << " " << fMax << endl;
139 }
140
141 Ssiz_t Match(const TString &str, Int_t idx, Long_t run=-1)
142 {
143 if (!IsValid(run))
144 return 0;
145
146 TString mods;
147 TArrayI pos;
148 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
149
150 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
151 }
152
153 Bool_t IsValid(ULong_t run) const { return run>=fMin && run<=fMax; }
154 ClassDef(Rule, 0)
155};
156ClassImp(Rule);
157
158class CheckMatch : public TObject
159{
160private:
161 TPRegexp fRunType1;
162 TPRegexp fRunType2;
163
164 TPRegexp fRegexp1;
165 TPRegexp fRegexp2;
166
167 ULong_t fMin;
168 ULong_t fMax;
169
170 void Init(const TObjArray &arr, Int_t idx=0)
171 {
172 const Int_t n = arr.GetEntries();
173
174 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
175
176 for (int i=0; i<n-idx; i++)
177 {
178 //cout << arr[i+idx]->GetName() << " ";
179
180 TString str(arr[i+idx]->GetName());
181 if (str=="*")
182 str = ".*";
183
184 switch (isminus && i>1 ? i+1 : i)
185 {
186 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
187 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
188 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
189 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
190 case 4: fMin = GetId(str); break;
191 case 5: fMax = GetId(str); break;
192 }
193 }
194 }
195
196public:
197 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
198
199 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
200 {
201 TObjArray *arr = txt.Tokenize(" ");
202 Init(*arr);
203 delete arr;
204 }
205
206 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
207 {
208 Init(arr, idx);
209 }
210
211 CheckMatch(const char *from, const char *to, Long_t min=0, Long_t max=-1)
212 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
213 {
214 }
215
216 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, ULong_t run=0)
217 {
218 if (run>0)
219 {
220 cout << " - - > " << run << endl;
221 if (run<fMin || run>fMax)
222 return kFALSE;
223 }
224
225 const TString test("X-"); // FIXME:STUPID!
226
227 //if (test.Index(fRunType2)==1)
228 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
229
230 if (test.Index(fRunType1,0)==1)
231 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
232
233 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
234 }
235 ClassDef(CheckMatch,0)
236};
237ClassImp(CheckMatch);
238
239class CheckList : public TList
240{
241public:
242 CheckList() { SetOwner(); }
243 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Long_t run=-1) const
244 {
245 TIter Next(this);
246
247 CheckMatch *check = 0;
248
249 while ((check=(CheckMatch*)Next()))
250 {
251 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
252 if (rc)
253 return rc;
254 }
255
256 return kFALSE;
257 }
258 ClassDef(CheckList,0)
259};
260ClassImp(CheckList);
261
262class SequenceBuild : public MSQLMagic
263{
264private:
265 TString fPathRawData;
266 TString fPathSequences;
267
268 Int_t fTelescopeNumber;
269
270 TMap fMap;
271 TList fListRegexp;
272
273 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
274 {
275 // Get the name of the column from result table
276 const TString key = res.GetFieldName(i);
277
278 // Get the list with the allowed attributed from the map
279 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
280 if (!list)
281 return kFALSE;
282
283 // Check whether the current run (row[0]) with the current attribute
284 // (row[i]) and the current run-type (row[1]) matches the attribute
285 // of the last run (keys[i] with run-type row[i])
286 return list->Matches(keys[1], row[1], keys[i], row[i], atol(row[0]));
287 }
288
289 Bool_t InsertSequence(std::pair<Long_t,Long_t> drs, Long_t from, Long_t to, const TString &cond)
290 {
291 //cout << " - Inserting Sequence into database." << endl;
292
293 TString query;
294 query = "SELECT fNight, STR_TO_DATE(fNight,'%Y%m%d') ";
295 query += " FROM RunInfo WHERE ";
296 query += cond;
297 query += Form(" AND fRunID=%ld", from);
298
299 TSQLResult *res = Query(query);
300 if (!res)
301 return kFALSE;
302
303 if (res->GetFieldCount()!=2 || res->GetRowCount()!=1)
304 {
305 cout << "ERROR - Wrong result from query: " << query << endl;
306 return kFALSE;
307 }
308
309 TSQLRow *row = res->Next();
310
311 UInt_t run = atoi((*row)[0]);
312 TString day = (*row)[1];
313
314 delete res;
315
316 query = "SELECT fRunID, fRunTypeKEY, fHasDrsFile, fDrsStep ";
317 query += " FROM RunInfo WHERE ";
318 query += cond;
319 query += Form(" AND fRunID BETWEEN %ld AND %ld ORDER BY fRunID", from, to);
320
321 if (!(res = Query(query)))
322 return kFALSE;
323
324 if (res->GetFieldCount()!=4)
325 {
326 cout << "ERROR - Wrong result from query: " << query << endl;
327 return kFALSE;
328 }
329
330 MSequence seq("sequence.def", "", (run%1000000)*1000+from);
331 seq.SetNight(day);
332
333 if (drs.first!=from)
334 {
335 seq.SetDrsSequence(drs.first);
336 if (drs.second!=-1)
337 seq.AddFile(drs.second, -1, MSequence::kDrsFile);
338 }
339
340 while ((row=res->Next()))
341 {
342 UInt_t num = atol((*row)[0]);
343 UInt_t key = atol((*row)[1]);
344
345 switch (key)
346 {
347 case 1: seq.AddFile(num, -1, MSequence::kDat); break;
348 case 2: seq.AddFile(num, -1, MSequence::kPed); break;
349 case 3: seq.AddFile(num, -1, MSequence::kDrsRun); break;
350 case 4: seq.AddFile(num, -1, MSequence::kCal); break;
351 case 5: seq.AddFile(num, -1, MSequence::kDat); break;
352 case 6: seq.AddFile(num, -1, MSequence::kCal); break;
353 }
354
355 if (atoi((*row)[2])==1)
356 if (!(*row)[3] || atoi((*row)[3])==2)
357 seq.AddFile(num, -1, MSequence::kDrsFile);
358 }
359
360 const TString path =
361 Form("%s/%04d/%02d/%02d", fPathSequences.Data(),
362 run/10000, (run/100)%100, run%100);
363
364 const TString fname =
365 Form("%s/%d_%03ld.%s", path.Data(), run, from,
366 drs.first==from?"drs.seq":"seq");
367
368 if (IsDummy())
369 {
370 cout << "----- " << fname << " -----" << endl;
371 seq.Print();
372 cout << "^^^^^ " << fname << " ^^^^^" << endl;
373 }
374 else
375 {
376 if (gSystem->AccessPathName(path, kFileExists))
377 if (gSystem->mkdir(path, kTRUE)<0)
378 cout << "ERROR - Could not create " << path << endl;
379
380 if (!gSystem->AccessPathName(path, kFileExists))
381 seq.WriteFile(fname);
382 }
383
384 delete res;
385
386 if (drs.first==from)
387 return kTRUE;
388
389 // ========== Request number of events ==========
390
391 // Can be replaced by
392
393 const TString runtime =
394 "SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))), ";
395
396 const TString where = Form("(fRunID BETWEEN %ld AND %ld) AND fExcludedFDAKEY=1 AND %s",
397 from, to, cond.Data());
398
399 query = "SELECT SUM(fNumEvents), ";
400 query += runtime;
401 query += " MIN(fZenithDistanceMin), MAX(fZenithDistanceMax), ";
402 query += " MIN(fAzimuthMin), MAX(fAzimuthMax), ";
403 query += " MIN(fRunStart), MAX(fRunStop) ";
404 query += " FROM RunInfo WHERE ";
405 query += where;
406 query += " AND fRunTypeKEY=1";// GROUP BY fNight";
407
408 res = Query(query);
409 if (!res)
410 return kFALSE;
411
412 row = res->Next();
413 if (!row || res->GetFieldCount()!=8)
414 {
415 cout << "ERROR - Wrong result from query: " << query << endl;
416 return kFALSE;
417 }
418
419 const TString nevts = (*row)[0];
420 const TString secs = (*row)[1];
421 const TString zdmin = (*row)[2];
422 const TString zdmax = (*row)[3];
423 const TString azmin = (*row)[4];
424 const TString azmax = (*row)[5];
425 const TString start = (*row)[6];
426 const TString stop = (*row)[7];
427
428 delete res;
429/*
430 const TString elts = GetELTSource(where);
431 if (elts.IsNull())
432 return kFALSE;
433
434 const TString eltp = GetELTProject(where);
435 if (eltp.IsNull())
436 return kFALSE;
437
438 // ========== Request data of sequence ==========
439 query = "SELECT ";
440
441 query += elts;
442 query += ", ";
443 query += eltp;
444
445 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
446 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
447 " fTriggerDelayTableKEY, fObservationModeKEY, fSumTriggerFlagKEY "
448 " FROM RunData WHERE fRunTypeKEY=2 AND ";
449 query += where;
450 query += " LIMIT 1";
451
452 res = Query(query);
453 if (!res)
454 return kFALSE;
455
456 row = res->Next();
457 if (!row || res->GetFieldCount()!=9)
458 {
459 cout << "ERROR - No result from query: " << query << endl;
460 return kFALSE;
461 }
462 */
463
464 const TString set1 = Form("fSequenceID=%ld", from);
465 const TString set2 = set1 + Form(", fNight=%d", run);
466 const TString set3 = set2 + Form(", fStartTime=Now(), fStopTime=Now(), fProcessingSiteKEY=4, fAvailable=Now() ");
467
468 TString query1;
469 query1 += set2+",";
470 query1 += Form(" fNumEvents=%s,", nevts.Data());
471 query1 += Form(" fSequenceDuration=%s,", secs.Data());
472 query1 += Form(" fSequenceStart=\"%s\",", start.Data());
473 query1 += Form(" fSequenceStop=\"%s\",", stop.Data());
474 query1 += Form(" fZenithDistanceMin=%s,", zdmin.IsNull() ? "NULL" : zdmin.Data());
475 query1 += Form(" fZenithDistanceMax=%s,", zdmax.IsNull() ? "NULL" : zdmax.Data());
476 query1 += Form(" fAzimuthMin=%s,", azmin.IsNull() ? "NULL" : azmin.Data());
477 query1 += Form(" fAzimuthMax=%s", azmax.IsNull() ? "NULL" : azmax.Data());
478
479 if (!Insert("SequenceInfo", query1))
480 {
481 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
482 return kFALSE;
483 }
484
485 if (!Insert("SequenceFileAvailISDCStatus", set3))
486 {
487 cout << "ERROR - Could not insert Sequence into SequenceFileAvailISDCStatus." << endl;
488 return kFALSE;
489 }
490
491 if (!Update("RunInfo", set1, where))
492 {
493 cout << "ERROR - Could not update RunData." << endl;
494 return kFALSE;
495 }
496
497 return kTRUE;
498 }
499
500 Bool_t DeleteSequence(Int_t sequ, UInt_t night)
501 {
502 if (fPathRawData.IsNull() || fPathSequences.IsNull())
503 {
504 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
505 return kTRUE;
506 }
507
508 //queries to delete information from the database
509 const TString cond(Form("fSequenceID=%d AND fNight=%d", sequ, night));
510
511 const TString path(Form("%04d/%02d/%02d", night/10000, (night/100)%100, night%100));
512
513 //commands to delete files from the disk
514 const TString cmd0(Form("rm -f %s/%s/%08d_%03d*seq", fPathSequences.Data(), path.Data(), night, sequ));
515
516// const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
517// const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
518
519 if (!Delete("SequenceInfo", cond))
520 return 2;
521
522 if (!Update("RunInfo", "fSequenceID=NULL", cond))
523 return 2;
524
525 if (IsDummy())
526 {
527 cout << " + would execute: " << cmd0 << endl;
528// cout << " + would execute: " << cmd1 << endl;
529// cout << " + would execute: " << cmd2 << endl;
530 return kTRUE;
531 }
532
533 cout << " + will execute: " << cmd0 << endl;
534// cout << " + will execute: " << cmd1 << endl;
535// cout << " + will execute: " << cmd2 << endl;
536
537 gSystem->Exec(cmd0);
538// gSystem->Exec(cmd1);
539// gSystem->Exec(cmd2);
540
541 return kTRUE;
542 }
543
544 Int_t CheckSequence(Long_t runstart, Long_t runstop, const TString &cond)
545 {
546 const char *fmt1 = "SELECT fRunID FROM RunInfo WHERE";
547 const char *fmt2 = "AND fExcludedFDAKEY=1 ORDER BY fRunID";
548
549 const TString query1 = Form("%s fSequenceID=%ld AND %s %s", fmt1, runstart, cond.Data(), fmt2);
550 const TString query2 = Form("%s fRunID BETWEEN %ld AND %ld AND %s %s", fmt1, runstart, runstop, cond.Data(), fmt2);
551
552 TSQLResult *res1 = Query(query1);
553 if (!res1)
554 return 2;
555
556 TSQLResult *res2 = Query(query2);
557 if (!res2)
558 {
559 delete res1;
560 return 2;
561 }
562
563 while (1)
564 {
565 TSQLRow *row1 = res1->Next();
566 TSQLRow *row2 = res2->Next();
567
568 if (!row1 && !row2)
569 return kTRUE;
570
571 if (!row1 || !row2)
572 return kFALSE;
573
574 if (atol((*row1)[0])!=atol((*row2)[0]))
575 return kFALSE;
576 }
577
578 return kFALSE;
579 }
580
581 Int_t CreateSequence(std::pair<Long_t,Long_t> &drs, Long_t runstart, Long_t runstop, const TString &cond)
582 {
583 cout << " * Creating Sequence " << runstart << "-" << runstop << " (drs=" << drs.first << "/" << drs.second << "):" << endl;
584
585 //----------------------------------------------------------------------------
586
587 TString query;
588 query = "SELECT fRunID FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
589 query += cond;
590 query += " AND fRunTypeKEY=4 AND fRunID BETWEEN ";
591 query += Form("%ld AND %ld", runstart, runstop);
592
593 TSQLResult *res = Query(query);
594 if (!res)
595 return kFALSE;
596
597 // Check if this is a DRS calibration (contains a G run)
598 if (res->GetRowCount()>0 && res->GetFieldCount()==1)
599 {
600 delete res;
601
602 // Get DRS file with pedestal (roi<1024)
603 query = "SELECT MAX(fRunID) FROM RunInfo WHERE ";
604 query += cond;
605 query += " AND fHasDrsFile=1 AND (fDrsStep IS NULL OR fDrsStep=2) AND fRunTypeKEY=2";
606 query += " AND fRunID BETWEEN ";
607 query += Form("%ld AND %ld", runstart, runstop);
608
609 res = Query(query);
610 if (!res)
611 return kFALSE;
612
613 if (res->GetRowCount()==0 || res->GetFieldCount()!=1)
614 return kFALSE;
615
616 TSQLRow *row = res->Next();
617
618 drs.first = runstart;
619 drs.second = (*row)[0] ? atol((*row)[0]) : -1;
620 }
621
622 delete res;
623
624 //----------------------------------------------------------------------------
625
626 query = "SELECT fSequenceID, fNight FROM RunInfo WHERE ";
627 query += cond;
628 query += " AND fRunID BETWEEN ";
629 query += Form("%ld AND %ld", runstart, runstop);
630 query += " AND fSequenceID>0";
631 query += " GROUP BY fSequenceID";
632
633 res = Query(query);
634 if (!res)
635 return -2;
636
637 const Int_t cnt = res->GetRowCount();
638
639 Int_t rc = kTRUE;
640 if (cnt==1)
641 {
642 TSQLRow *row=res->Next();
643 const Int_t check = CheckSequence(runstart, runstop, cond);
644 if (check==kTRUE)
645 {
646 cout << " - Identical sequence already existing." << endl;
647 delete res;
648 return -1; // kTRUE
649 }
650 if (check==2)
651 rc=2;
652 else
653 {
654 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
655 if (DeleteSequence(atoi((*row)[0]), atoi((*row)[1]))==2)
656 rc = 2;
657 }
658 }
659 else
660 {
661 TSQLRow *row=0;
662 while ((row=res->Next()))
663 {
664 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
665 if (DeleteSequence(atoi((*row)[0]), atoi((*row)[1]))==2)
666 rc = 2;
667 }
668 }
669
670 delete res;
671
672 if (rc==2)
673 return -2;
674
675 if (!InsertSequence(drs, runstart, runstop, cond))
676 return -2;
677
678 return kTRUE;
679 }
680
681 Bool_t ReadResources(const char *fname)
682 {
683 // Check for the section header
684 TPRegexp regexp("^\\[(RunTypes|Transition|Regexp):?[0-9 ]*\\]$");
685 // Check if section header contains a number
686 TPRegexp regnum("[0-9]");
687 // Check if section header contains the telescope number
688 TPRegexp regtel(Form("[^0-9]0*%d[^0-9]", fTelescopeNumber));
689
690 ifstream fin(fname);
691 if (!fin)
692 {
693 cout << "Cannot open file " << fname << ": ";
694 cout << strerror(errno) << endl;
695 return kFALSE;
696 }
697
698 Int_t section = 0;
699
700 TString key;
701 while (1)
702 {
703 TString txt;
704 txt.ReadLine(fin);
705 if (!fin)
706 break;
707
708 txt = txt.Strip(TString::kBoth);
709
710 if (txt[0]=='#' || txt.IsNull())
711 continue;
712
713 if (txt[0]=='[')
714 {
715 TString sec = txt(regexp);
716 if (!sec.IsNull())
717 {
718 // Skip sections with the wrong telescope number
719 // (If section contains numbers but not the tel num)
720 if (!sec(regnum).IsNull() && sec(regtel).IsNull())
721 continue;
722
723 // Check which section we are in
724 if (sec.BeginsWith("[RunTypes"))
725 section = 1;
726 if (sec.BeginsWith("[Transition"))
727 section = 2;
728 if (sec.BeginsWith("[Regexp"))
729 section = 3;
730
731 continue;
732 }
733
734 if (section!=3)
735 {
736 cout << "WARNING - Line starts with [ but we are not in the Regexp section." << endl;
737 cout << txt << endl;
738 continue;
739 }
740 }
741
742
743 TObjArray *arr = txt.Tokenize(" ");
744
745 if (arr->GetEntries()>0)
746 switch (section)
747 {
748 case 1:
749 // Create a llist of run-types
750 // drs-time T
751 break;
752 case 2:
753 {
754 TString key = arr->At(0)->GetName();
755 key.Prepend("f");
756 key.Append("KEY");
757
758 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
759 if (!list)
760 {
761 //cout << key << endl;
762 list = new CheckList;
763 fMap.Add(new TObjString(key), list);
764 }
765
766 if (arr->GetEntries()>1)
767 {
768 //cout << key << " ";
769 list->Add(new CheckMatch(*arr, 1));
770 }
771 }
772 break;
773 case 3:
774 fListRegexp.Add(new Rule(*arr));
775 break;
776 }
777
778 delete arr;
779 }
780
781 return kTRUE;
782 }
783/*
784 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
785 {
786 TObjArray names; // array with old names (including regexp)
787
788 // Add to array and expand the array if necessary
789 TSQLRow *row=0;
790 while ((row=res->Next()))
791 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
792
793 // Now a LUT is build which converts the keys for
794 // the names including the regexp into keys for
795 // the names excluding the regexp
796 TString elt(Form("ELT(RunData.f%sKEY+1", col));
797
798 // loop over all entries in the list
799 const Int_t n = names.GetSize();
800 for (int i=0; i<n; i++)
801 {
802 // For all entries which are not in the list
803 // write an undefined value into the LUT
804 TObject *o = names.UncheckedAt(i);
805 if (!o)
806 {
807 elt += ",0";
808 continue;
809 }
810
811 // Remove the regexp from the string which includes it
812 TString name = o->GetName();
813
814 TIter NextR(&regexp);
815 TObject *obj=0;
816 while ((obj=NextR()))
817 {
818 TPRegexp reg(obj->GetName());
819 const Ssiz_t pos = name.Index(reg, 0);
820 if (pos>0)
821 {
822 name.Remove(pos);
823 name += "-W";
824 break;
825 }
826 }
827
828 // Check if such a Key exists, if not insert it
829 const Int_t key = QueryKeyOfName(col, name);
830
831 // RESOLVE return code!!!
832 //if (key<0)
833 // return "";
834
835 // add index to the LUT
836 elt += Form(",%d", key);
837 }
838
839 // close LUT expression
840 // elt += ") AS Elt";
841 // elt += col;
842
843 elt += ") AS f";
844 elt += col;
845 elt += "KEY";
846
847 // return result
848 return elt;
849 }
850
851 TString GetELT(const char *col, TSQLResult *res, TString regexp)
852 {
853 TList list;
854 list.SetOwner();
855 list.Add(new TObjString(regexp));
856 return GetELT(col, res, list);
857 }
858
859 TString GetELTQuery(const char *col, const char *cond) const
860 {
861 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
862 "LEFT JOIN %s USING (f%sKEY) WHERE %s GROUP BY f%sName",
863 col, col, col, col, cond, col);
864 }
865
866 TString GetELTSource(const char *cond)
867 {
868 //query all sources observed in this night
869 TSQLResult *resx = Query(GetELTQuery("Source", cond));
870 if (!resx)
871 return "";
872
873 // In the case there is only a single source
874 // do not replace the source key by the ELT
875 if (resx->GetRowCount()==1)
876 return "fSourceKEY";
877
878 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
879 delete resx;
880
881 return elts;
882 }
883
884 TString GetELTProject(const char *cond)
885 {
886 //query all project names observed in this night
887 TSQLResult *resx = Query(GetELTQuery("Project", cond));
888 if (!resx)
889 return "";
890
891 // In the case there is only a single project
892 // do not replace the project key by the ELT
893 if (resx->GetRowCount()==1)
894 return "fProjectKEY";
895
896 TList regexp;
897 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
898 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
899
900 TString eltp2 = GetELT("Project", resx, regexp);
901 delete resx;
902 regexp.Delete();
903
904 return eltp2;
905 }
906*/
907 Bool_t HasAtLeastOne(TString src, TString chk) const
908 {
909 src.ToLower();
910 chk.ToLower();
911
912 for (int i=0; i<chk.Length(); i++)
913 if (src.First(chk[i])<0)
914 return kFALSE;
915
916 return kTRUE;
917 }
918
919 TString PrepareString(TSQLResult &res, TArrayL &runs, TArrayL &nights)
920 {
921 // Number of result rows
922 const Int_t rows = res.GetRowCount();
923
924 runs.Set(rows); // initialize size of array for run numbers
925 nights.Set(rows); // initialize size of array for nights
926
927 TArrayD start(rows); // instantiate array for start times
928 TArrayD stop(rows); // instantiate array for stop times
929
930 TString str; // result string
931
932 Int_t idx=0;
933 TSQLRow *row=0;
934 while ((row=res.Next()))
935 {
936 runs[idx] = atol((*row)[0]); // run number
937 nights[idx] = atol((*row)[4]); // night
938
939 const TString tstart = ((*row)[2]); // start time
940 const TString tstop = ((*row)[3]); // stop time
941
942 start[idx] = MTime(tstart).GetMjd(); // convert to double
943 stop[idx] = MTime(tstop).GetMjd(); // convert to double
944
945 // This is a workaround for broken start-times
946 if (tstart=="0000-00-00 00:00:00")
947 start[idx] = stop[idx];
948
949 // Add a run-type character for this run to the string
950 str += RunType((*row)[1]);
951
952 // Increase index
953 idx++;
954 }
955/*
956 // Now the P- and D- runs are classified by the time-distance
957 // to the next or previous C-Run
958 Double_t lastc = -1;
959 Double_t nextc = -1;
960 for (int i=0; i<str.Length(); i++)
961 {
962 if (str[i]=='C')
963 {
964 // Remember stop time of C-Run as time of last C-run
965 lastc = stop[i];
966
967 // Calculate start time of next C-Run
968 const TString residual = str(i+1, str.Length());
969 const Int_t pos = residual.First('C');
970
971 // No C-Run found anymore. Finished...
972 if (pos<0)
973 break;
974
975 // Remember start time of C-Run as time of next C-run
976 nextc = start[i+1+pos];
977 continue;
978 }
979
980 // Check whether the identifying character should
981 // be converted to lower case
982 if (start[i]-lastc>nextc-stop[i])
983 str[i] = tolower(str[i]);
984 }
985*/
986 //cout << str << endl;
987 return str;
988 }
989
990 void PrintResidual(Long_t runstart, Long_t runstop, TString residual, const char *descr)
991 {
992 residual.ToLower();
993
994 // Count number of unsequences "characters"
995 const Int_t numd = residual.CountChar('d');
996 const Int_t nump = residual.CountChar('p');
997 const Int_t numb = residual.CountChar('b');
998 const Int_t numg = residual.CountChar('g');
999 const Int_t numt = residual.CountChar('t');
1000 const Int_t numc = residual.CountChar('c');
1001
1002 // Print some information to the output steram
1003 if (numd+nump+numb+numg+numt+numc==0)
1004 return;
1005
1006 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
1007 if (numd>0)
1008 cout << " (DATA!)";
1009/*
1010 if (nump==0 || numc==0 || numd==0)
1011 cout << " Missing";
1012 if (numd==0)
1013 cout << " D";
1014 if (nump==0)
1015 cout << " P";
1016 if (numc==0)
1017 cout << " C";
1018*/
1019 cout << endl;
1020 }
1021
1022 Int_t SplitBlock(Long_t runstart, Long_t runstop, const TString &cond)
1023 {
1024 // Request data necessary to split block into sequences
1025 /*const*/
1026 TString query =
1027 "SELECT fRunID, fRunTypeKEY, fRunStart, fRunStop, fNight*1000 "
1028 " FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
1029 query += Form("fRunID BETWEEN %ld AND %ld AND "
1030 /*" (fRunTypeKEY BETWEEN 2 AND 4)"*/, runstart, runstop);
1031
1032 query += cond;
1033 query += " ORDER BY fRunID";
1034
1035
1036 // Send query
1037 TSQLResult *res = Query(query);
1038 if (!res)
1039 return 2;
1040
1041 // Get String containing the sequence of P-,C- and D-Runs
1042 // and an array with the corresponding run-numbers
1043 TArrayL runs;
1044 TArrayL nights;
1045 const TString str = PrepareString(*res, runs, nights);
1046
1047 delete res;
1048
1049 // Check if the prepared string at least contains one run of each type
1050 // ??????????????????????????????????
1051 /*
1052 if (!HasAtLeastOne(str, "PCD"))
1053 {
1054 PrintResidual(runstart, runstop, str, "Block");
1055 return kTRUE;
1056 }*/
1057 // ??????????????????????????????????
1058
1059 // Start the sequence building
1060 // ---------------------------
1061
1062 Int_t pos = 0; // Position in the string
1063 std::map<Long_t, Long_t> map; // Map for resulting sequences
1064 TString residual; // Unsequences "characters"
1065
1066 // Step through the string one character by one
1067 cout << " ";
1068 while (pos<str.Length())
1069 {
1070 Bool_t found = kFALSE;
1071
1072 // Loop over the predefined regular expressions
1073 TIter Next(&fListRegexp);
1074 Rule *obj = 0;
1075 while ((obj=(Rule*)Next()))
1076 {
1077 // Check if regular expressions matchs
1078 const Ssiz_t len = obj->Match(str, pos, nights[pos]);
1079 if (len>0)
1080 {
1081 // In case of match get the substring which
1082 // defines the sequence and print it
1083 TString sub = str(pos, len);
1084 sub.ToUpper();
1085 cout << runs[pos]<<":"<< sub << "|";
1086
1087 // Add the first and last run of the sequence to the map
1088 //map.Add(runs[pos], runs[pos+len-1]);
1089 map[runs[pos]] = runs[pos+len-1];
1090
1091 // A sequence was found...
1092 found = kTRUE;
1093
1094 // step forward in the string by the length of the sequence
1095 pos += len;
1096 break;
1097 }
1098 }
1099
1100 // If a sequence was found go on...
1101 if (found)
1102 continue;
1103
1104 // print unsequenced characters
1105 cout << (char)tolower(str[pos]);
1106
1107 // Count the number of "characters" not sequenced
1108 residual += str[pos];
1109
1110 // step one character forward
1111 pos++;
1112 }
1113 cout << endl;
1114
1115 PrintResidual(runstart, runstop, residual, "Runs ");
1116
1117 // Create all sequences which were previously found
1118 std::pair<Long_t,Long_t> drs(-1, -1);
1119 for (std::map<Long_t,Long_t>::const_iterator it=map.begin();
1120 it!=map.end(); it++)
1121 if (CreateSequence(drs, it->first, it->second, cond)==2)
1122 return 2;
1123 /*
1124 Long64_t first, last;
1125 TExMapIter iter(&map);
1126 while (iter.Next(first, last))
1127 if (CreateSequence(first, last, cond)==2)
1128 return 2;
1129 */
1130 return kTRUE;
1131 }
1132
1133 Char_t RunType(Int_t n) const
1134 {
1135 switch (n)
1136 {
1137 case 1: return 'D';
1138 case 2: return 'P'; // Ped1024
1139 case 3: return 'B'; // Ped 300
1140 case 4: return 'G'; // drs-gain
1141 case 5: return 'T'; // drs-time
1142 case 6: return 'C'; // light-pulser-ext
1143 case 11: return 'M'; // ped-and-lp-ext
1144 case 12: return 'U'; // drs-time-upshifted
1145 }
1146 return '-';
1147 }
1148 Char_t RunType(const char *str) const
1149 {
1150 return RunType(atoi(str));
1151 }
1152
1153 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks, const TString &cond)
1154 {
1155 // col key content
1156 // -----------------------------
1157 // 0 - runnumber
1158 // 1 0 RunTypeKEY
1159 // 2 1 source
1160 // 3 2 project
1161 // . .
1162 // . .
1163 // . . from transition.txt
1164 //
1165
1166 //build blocks of runs, which have the same values
1167 //for each block the first and the last run are stored in a TExMap
1168 //the values are checked with the help of an array of TStrings
1169 Long_t runstart = -1;
1170 Long_t runstop = -1;
1171
1172 const UInt_t ncols = res->GetFieldCount();
1173
1174 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
1175
1176 // Loop over runs
1177 TSQLRow *row=0;
1178 while ((row=res->Next()))
1179 {
1180 // This is the runnumber of the first run in the new block
1181 if (runstart<0)
1182 runstart=atol((*row)[0]);
1183
1184 // Check which transitions might be allowed for this run and
1185 // which are not. Check also which attributes should be ignored
1186 // for this run. The information about it is stored in the map.
1187 Int_t rc[ncols];
1188 for (UInt_t i=2; i<ncols; i++)
1189 rc[i] = CheckTransition(*res, keys, *row, i);
1190
1191 for (UInt_t i=2; i<ncols; i++)
1192 {
1193 switch (rc[i])
1194 {
1195 case kTRUE: // Transition allowed, switch to new attribute
1196 keys[i] = (*row)[i];
1197 continue;
1198
1199 case -1: // Ignore attribute, continue with next one
1200 //if (keys[i] == (*row)[i])
1201 // break; // Only ignore it if it is really different
1202 continue;
1203
1204 case kFALSE: // No decision (nothing matching found in map)
1205 break; // go on checking
1206
1207 }
1208
1209 // If past-attribute is not yet initialized, do it now.
1210 if (keys[i].IsNull())
1211 keys[i] = (*row)[i];
1212
1213 // Check whether key has changed for this run
1214 // (this condition is never true for the first run)
1215 // This condition in only reached if keys[i] was initialized
1216 if (keys[i] == (*row)[i])
1217 continue;
1218
1219 // Found one block with unique keys, fill values into TExMap
1220 // (except if this is the first run processed)
1221 cout << endl;
1222 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1223 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1224 if (SplitBlock(runstart, runstop, cond)==2)
1225 return 2;
1226 cout << " - Transition from " << RunType(keys[1]) << ":";
1227 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1228 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1229 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1230
1231 // This is already the first run of the new block
1232 runstart=atol((*row)[0]);
1233
1234 // These are the keys corresponding to the first run
1235 // in the new block. All following runs should have
1236 // identical keys. Do not set the attribute if for
1237 // this attribute the transition check evaluated
1238 // to "ignore".
1239 for (UInt_t j=2; j<ncols; j++)
1240 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1241
1242 break;
1243 }
1244
1245 keys[1] = (*row)[1]; // Remember run type for next run
1246
1247 // This is the new runnumber of the last run in this block
1248 runstop=atol((*row)[0]);
1249 }
1250
1251 // If runstart==-1 a possible block would contain a single run...
1252 if (runstart>0 && runstart!=runstop)
1253 {
1254 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1255 //fill values into TExMap (last value)
1256 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1257 if (SplitBlock(runstart, runstop, cond)==2)
1258 return 2;
1259 }
1260 cout << "Done." << endl << endl;
1261
1262 return kTRUE;
1263 }
1264
1265public:
1266 SequenceBuild(Int_t tel=1, const char *rc="sql.rc") : MSQLMagic(rc), fTelescopeNumber(tel)
1267 {
1268 fListRegexp.SetOwner();
1269
1270 // FIXME: THIS IS NOT YET HANDLED
1271 if (ReadResources("fact/resources/sequences.rc"))
1272 return;
1273 }
1274
1275 SequenceBuild(TEnv &env, Int_t tel=1) : MSQLMagic(env), fTelescopeNumber(tel)
1276 {
1277 fListRegexp.SetOwner();
1278
1279 // FIXME: THIS IS NOT YET HANDLED
1280 if (ReadResources("fact/resources/sequences.rc"))
1281 return;
1282 }
1283 ~SequenceBuild()
1284 {
1285 fMap.DeleteAll();
1286 }
1287
1288 void SetPathRawData(const char *path) { fPathRawData=path; }
1289 void SetPathSequences(const char *path) { fPathSequences=path; }
1290
1291 int Build(TString day)
1292 {
1293 cout << endl;
1294 cout << "Night of sunset at " << day << ":" << endl;
1295
1296 const TString cond = Form("fNight=%s ", day.Data());
1297
1298 // Form("fRunNumber=%s "
1299 // "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 "
1300 // "AND fTelescopeNumber=%d ",
1301 // day.Data(), fTelescopeNumber);
1302/*
1303 //query all sources observed in this night
1304 const TString elts = GetELTSource(cond);
1305 if (elts.IsNull())
1306 return 2;
1307
1308 //query all project names observed in this night
1309 const TString eltp2 = GetELTProject(cond);
1310 if (elts.IsNull())
1311 return 2;
1312*/
1313 // Setup query to get all values from the database,
1314 // that are relevant for building sequences
1315 TString query("SELECT fRunID, fRunTypeKEY ");
1316/*
1317 query += elts;
1318 query += ", ";
1319 query += eltp2;
1320*/
1321 // Now add all entries from the transition table to the query
1322 TIter NextPair(&fMap);
1323 TObject *mapkey = 0;
1324 while ((mapkey=(TPair*)NextPair()))
1325 if (!query.Contains(mapkey->GetName()))
1326 {
1327 query += ", ";
1328 query += mapkey->GetName();
1329 }
1330 query += " FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
1331 query += cond;
1332 query += " ORDER BY fRunID";
1333
1334 TSQLResult *res = Query(query);
1335 if (!res)
1336 return 2;
1337
1338 TExMap blocks;
1339 const Int_t rc = BuildBlocks(res, blocks, cond);
1340 delete res;
1341
1342 return rc;
1343 }
1344
1345 Int_t Build()
1346 {
1347 //get all dates from the database
1348 TSQLResult *res = Query("SELECT fNight FROM RunInfo GROUP BY fNight ORDER BY fNight DESC");
1349 if (!res)
1350 return 2;
1351
1352 //execute buildsequenceentries for all dates
1353 TSQLRow *row=0;
1354 while ((row=res->Next()))
1355 Build((*row)[0]);
1356
1357 delete res;
1358
1359 return 1;
1360 }
1361 ClassDef(SequenceBuild, 0)
1362};
1363
1364ClassImp(SequenceBuild);
1365
1366int buildseqentries(TString day, TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1367{
1368 SequenceBuild serv(tel, "sql.rc");
1369 if (!serv.IsConnected())
1370 {
1371 cout << "ERROR - Connection to database failed." << endl;
1372 return 0;
1373 }
1374
1375 cout << "buildseqentries" << endl;
1376 cout << "---------------" << endl;
1377 cout << endl;
1378 cout << "Connected to " << serv.GetName() << endl;
1379 if (!datapath.IsNull())
1380 cout << "DataPath: " << datapath << endl;
1381 if (!sequpath.IsNull())
1382 cout << "SeqPath: " << sequpath << endl;
1383 cout << "Day: " << day << endl;
1384 cout << "Telescope: " << tel << endl;
1385 cout << endl;
1386
1387 serv.SetIsDummy(dummy);
1388 serv.SetPathRawData(datapath);
1389 serv.SetPathSequences(sequpath);
1390 return serv.Build(day);
1391}
1392
1393//
1394// Build Sequences for all Nights
1395//
1396int buildseqentries(TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1397{
1398 SequenceBuild serv(tel, "sql.rc");
1399 if (!serv.IsConnected())
1400 {
1401 cout << "ERROR - Connection to database failed." << endl;
1402 return 0;
1403 }
1404
1405 cout << "buildseqentries" << endl;
1406 cout << "---------------" << endl;
1407 cout << endl;
1408 cout << "Connected to " << serv.GetName() << endl;
1409 cout << "DataPath: " << datapath << endl;
1410 cout << "SeqPath: " << sequpath << endl;
1411 cout << "Telescope: " << tel << endl;
1412 cout << endl;
1413
1414 serv.SetIsDummy(dummy);
1415 serv.SetPathRawData(datapath);
1416 serv.SetPathSequences(sequpath);
1417 return serv.Build();
1418}
1419
1420int buildseqentries(Int_t tel=1, Bool_t dummy=kTRUE)
1421{
1422 return buildseqentries("", "", tel, dummy);
1423}
1424
1425int buildseqentries(TString day, Int_t tel=1, Bool_t dummy=kTRUE)
1426{
1427 return buildseqentries(day, "", "", tel, dummy);
1428}
Note: See TracBrowser for help on using the repository browser.