source: trunk/Mars/fact/processing/buildseqentries.C@ 19337

Last change on this file since 19337 was 18308, checked in by Daniela Dorner, 10 years ago
bugfix (start and stop in regular expressions of sequences.rc were ignored)
File size: 44.5 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82
83#include <map>
84#include <utility>
85
86#include <errno.h>
87
88#include <TSQLRow.h>
89#include <TSQLResult.h>
90
91#include <TEnv.h>
92#include <TMap.h>
93#include <TMath.h>
94#include <TExMap.h>
95#include <TArrayI.h>
96#include <TArrayL.h>
97#include <TArrayD.h>
98#include <TPRegexp.h>
99#include <TSystem.h>
100#include <TObjString.h>
101#include <TObjArray.h>
102
103#include "MTime.h"
104#include "MSequence.h"
105
106#include "MSQLMagic.h"
107
108using namespace std;
109
110 /*static*/ ULong_t GetId(const TString str)
111 {
112 const Ssiz_t dot = str.First('.');
113
114 const UInt_t run = str.Atoi();
115 const UInt_t sub = dot<0 ? 0 : atoi(str.Data()+dot+1);
116
117 return ULong_t(run)*1000+sub;
118 }
119
120
121class Rule : public TObject
122{
123private:
124 TPRegexp fRegexp;
125
126 ULong_t fMin;
127 ULong_t fMax;
128
129public:
130 Rule(TObjArray &arr) :
131 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
132 fMin(0), fMax((ULong_t)-1)
133 {
134 if (arr.GetEntries()>1)
135 fMin = GetId(arr[1]->GetName());
136 if (arr.GetEntries()>2)
137 fMax = GetId(arr[2]->GetName());
138 cout << "----> " << fMin << " " << fMax << endl;
139 }
140
141 Ssiz_t Match(const TString &str, Int_t idx, Long_t run=-1)
142 {
143 if (!IsValid(run))
144 return 0;
145
146 TString mods;
147 TArrayI pos;
148 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
149
150 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
151 }
152
153 Bool_t IsValid(ULong_t run) const { return run>=fMin && run<=fMax; }
154 ClassDef(Rule, 0)
155};
156ClassImp(Rule);
157
158class CheckMatch : public TObject
159{
160private:
161 TPRegexp fRunType1;
162 TPRegexp fRunType2;
163
164 TPRegexp fRegexp1;
165 TPRegexp fRegexp2;
166
167 ULong_t fMin;
168 ULong_t fMax;
169
170 void Init(const TObjArray &arr, Int_t idx=0)
171 {
172 const Int_t n = arr.GetEntries();
173
174 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
175
176 for (int i=0; i<n-idx; i++)
177 {
178 //cout << arr[i+idx]->GetName() << " ";
179
180 TString str(arr[i+idx]->GetName());
181 if (str=="*")
182 str = ".*";
183
184 switch (isminus && i>1 ? i+1 : i)
185 {
186 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
187 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
188 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
189 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
190 case 4: fMin = GetId(str); break;
191 case 5: fMax = GetId(str); break;
192 }
193 }
194 }
195
196public:
197 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
198
199 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
200 {
201 TObjArray *arr = txt.Tokenize(" ");
202 Init(*arr);
203 delete arr;
204 }
205
206 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
207 {
208 Init(arr, idx);
209 }
210
211 CheckMatch(const char *from, const char *to, Long_t min=0, Long_t max=-1)
212 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
213 {
214 }
215
216 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, ULong_t run=0)
217 {
218 if (run>0)
219 {
220 cout << " - - > " << run << endl;
221 if (run<fMin || run>fMax)
222 return kFALSE;
223 }
224
225 const TString test("X-"); // FIXME:STUPID!
226
227 //if (test.Index(fRunType2)==1)
228 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
229
230 if (test.Index(fRunType1,0)==1)
231 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
232
233 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
234 }
235 ClassDef(CheckMatch,0)
236};
237ClassImp(CheckMatch);
238
239class CheckList : public TList
240{
241public:
242 CheckList() { SetOwner(); }
243 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Long_t run=-1) const
244 {
245 TIter Next(this);
246
247 CheckMatch *check = 0;
248
249 while ((check=(CheckMatch*)Next()))
250 {
251 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
252 if (rc)
253 return rc;
254 }
255
256 return kFALSE;
257 }
258 ClassDef(CheckList,0)
259};
260ClassImp(CheckList);
261
262class SequenceBuild : public MSQLMagic
263{
264private:
265 TString fPathRawData;
266 TString fPathSequences;
267
268 Int_t fTelescopeNumber;
269
270 TMap fMap;
271 TList fListRegexp;
272
273 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
274 {
275 // Get the name of the column from result table
276 const TString key = res.GetFieldName(i);
277
278 // Get the list with the allowed attributed from the map
279 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
280 if (!list)
281 return kFALSE;
282
283 // Check whether the current run (row[0]) with the current attribute
284 // (row[i]) and the current run-type (row[1]) matches the attribute
285 // of the last run (keys[i] with run-type row[i])
286 return list->Matches(keys[1], row[1], keys[i], row[i], atol(row[0]));
287 }
288
289 Bool_t InsertSequence(std::pair<Long_t,Long_t> drs, Long_t from, Long_t to, const TString &cond)
290 {
291 //cout << " - Inserting Sequence into database." << endl;
292
293 TString query;
294 query = "SELECT fNight, STR_TO_DATE(fNight,'%Y%m%d') ";
295 query += " FROM RunInfo WHERE ";
296 query += cond;
297 query += Form(" AND fRunID=%ld", from);
298
299 TSQLResult *res = Query(query);
300 if (!res)
301 return kFALSE;
302
303 if (res->GetFieldCount()!=2 || res->GetRowCount()!=1)
304 {
305 cout << "ERROR - Wrong result from query: " << query << endl;
306 return kFALSE;
307 }
308
309 TSQLRow *row = res->Next();
310
311 UInt_t run = atoi((*row)[0]);
312 TString day = (*row)[1];
313
314 delete res;
315
316 query = "SELECT fRunID, fRunTypeKEY, fHasDrsFile, fDrsStep ";
317 query += " FROM RunInfo WHERE ";
318 query += cond;
319 query += Form(" AND fRunID BETWEEN %ld AND %ld ORDER BY fRunID", from, to);
320
321 if (!(res = Query(query)))
322 return kFALSE;
323
324 if (res->GetFieldCount()!=4)
325 {
326 cout << "ERROR - Wrong result from query: " << query << endl;
327 return kFALSE;
328 }
329
330 MSequence seq("sequence.def", "", (run%1000000)*1000+from);
331 seq.SetNight(day);
332
333 if (drs.first!=from)
334 {
335 seq.SetDrsSequence(drs.first);
336 if (drs.second!=-1)
337 seq.AddFile(drs.second, -1, MSequence::kDrsFile);
338 }
339
340 while ((row=res->Next()))
341 {
342 UInt_t num = atol((*row)[0]);
343 UInt_t key = atol((*row)[1]);
344
345 switch (key)
346 {
347 case 1: seq.AddFile(num, -1, MSequence::kDat); break;
348 case 2: seq.AddFile(num, -1, MSequence::kPed); break;
349 case 3: seq.AddFile(num, -1, MSequence::kDrsRun); break;
350 case 4: seq.AddFile(num, -1, MSequence::kCal); break;
351 case 5: seq.AddFile(num, -1, MSequence::kDat); break;
352 case 6: seq.AddFile(num, -1, MSequence::kCal); break;
353 }
354
355 if (atoi((*row)[2])==1)
356 if (!(*row)[3] || atoi((*row)[3])==2)
357 seq.AddFile(num, -1, MSequence::kDrsFile);
358 }
359
360 const TString path =
361 Form("%s/%04d/%02d/%02d", fPathSequences.Data(),
362 run/10000, (run/100)%100, run%100);
363
364 const TString fname =
365 Form("%s/%d_%03ld.%s", path.Data(), run, from,
366 drs.first==from?"drs.seq":"seq");
367
368 if (IsDummy())
369 {
370 cout << "----- " << fname << " -----" << endl;
371 seq.Print();
372 cout << "^^^^^ " << fname << " ^^^^^" << endl;
373 }
374 else
375 {
376 if (gSystem->AccessPathName(path, kFileExists))
377 if (gSystem->mkdir(path, kTRUE)<0)
378 cout << "ERROR - Could not create " << path << endl;
379
380 if (!gSystem->AccessPathName(path, kFileExists))
381 seq.WriteFile(fname);
382 }
383
384 delete res;
385
386 if (drs.first==from)
387 return kTRUE;
388
389 // ========== Request number of events ==========
390
391 // Can be replaced by
392
393 const TString runtime =
394 "SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))), ";
395
396 const TString where = Form("(fRunID BETWEEN %ld AND %ld) AND fExcludedFDAKEY=1 AND %s",
397 from, to, cond.Data());
398
399 query = "SELECT SUM(fNumEvents), ";
400 query += runtime;
401 query += " MIN(fZenithDistanceMin), MAX(fZenithDistanceMax), ";
402 query += " MIN(fAzimuthMin), MAX(fAzimuthMax), ";
403 query += " MIN(fRunStart), MAX(fRunStop) ";
404 query += " FROM RunInfo WHERE ";
405 query += where;
406 query += " AND fRunTypeKEY=1";// GROUP BY fNight";
407
408 res = Query(query);
409 if (!res)
410 return kFALSE;
411
412 row = res->Next();
413 if (!row || res->GetFieldCount()!=8)
414 {
415 cout << "ERROR - Wrong result from query: " << query << endl;
416 return kFALSE;
417 }
418
419 const TString nevts = (*row)[0];
420 const TString secs = (*row)[1];
421 const TString zdmin = (*row)[2];
422 const TString zdmax = (*row)[3];
423 const TString azmin = (*row)[4];
424 const TString azmax = (*row)[5];
425 const TString start = (*row)[6];
426 const TString stop = (*row)[7];
427
428 delete res;
429/*
430 const TString elts = GetELTSource(where);
431 if (elts.IsNull())
432 return kFALSE;
433
434 const TString eltp = GetELTProject(where);
435 if (eltp.IsNull())
436 return kFALSE;
437
438 // ========== Request data of sequence ==========
439 query = "SELECT ";
440
441 query += elts;
442 query += ", ";
443 query += eltp;
444
445 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
446 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
447 " fTriggerDelayTableKEY, fObservationModeKEY, fSumTriggerFlagKEY "
448 " FROM RunData WHERE fRunTypeKEY=2 AND ";
449 query += where;
450 query += " LIMIT 1";
451
452 res = Query(query);
453 if (!res)
454 return kFALSE;
455
456 row = res->Next();
457 if (!row || res->GetFieldCount()!=9)
458 {
459 cout << "ERROR - No result from query: " << query << endl;
460 return kFALSE;
461 }
462 */
463
464 const TString set1 = Form("fSequenceID=%ld", from);
465 const TString set2 = set1 + Form(", fNight=%d", run);
466 const TString set3 = set2 + Form(", fStartTime=Now(), fStopTime=Now(), fProcessingSiteKEY=4, fAvailable=Now() ");
467
468 TString query1;
469 query1 += set2+",";
470 query1 += Form(" fNumEvents=%s,", nevts.Data());
471 query1 += Form(" fSequenceDuration=%s,", secs.Data());
472 query1 += Form(" fSequenceStart=\"%s\",", start.Data());
473 query1 += Form(" fSequenceStop=\"%s\",", stop.Data());
474 query1 += Form(" fZenithDistanceMin=%s,", zdmin.IsNull() ? "NULL" : zdmin.Data());
475 query1 += Form(" fZenithDistanceMax=%s,", zdmax.IsNull() ? "NULL" : zdmax.Data());
476 query1 += Form(" fAzimuthMin=%s,", azmin.IsNull() ? "NULL" : azmin.Data());
477 query1 += Form(" fAzimuthMax=%s", azmax.IsNull() ? "NULL" : azmax.Data());
478
479 if (!Insert("SequenceInfo", query1))
480 {
481 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
482 return kFALSE;
483 }
484
485 if (!Insert("SequenceFileAvailISDCStatus", set3))
486 {
487 cout << "ERROR - Could not insert Sequence into SequenceFileAvailISDCStatus." << endl;
488 return kFALSE;
489 }
490
491 if (!Update("RunInfo", set1, where))
492 {
493 cout << "ERROR - Could not update RunData." << endl;
494 return kFALSE;
495 }
496
497 return kTRUE;
498 }
499
500 Bool_t DeleteSequence(Int_t sequ, UInt_t night)
501 {
502 if (fPathRawData.IsNull() || fPathSequences.IsNull())
503 {
504 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
505 return kTRUE;
506 }
507
508 //queries to delete information from the database
509 const TString cond(Form("fSequenceID=%d AND fNight=%d", sequ, night));
510
511 const TString path(Form("%04d/%02d/%02d", night/10000, (night/100)%100, night%100));
512
513 //commands to delete files from the disk
514 const TString cmd0(Form("rm -f %s/%s/%08d_%03d*seq", fPathSequences.Data(), path.Data(), night, sequ));
515
516// const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
517// const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
518
519 if (!Delete("SequenceInfo", cond))
520 return 2;
521
522 if (!Update("RunInfo", "fSequenceID=NULL", cond))
523 return 2;
524
525 if (IsDummy())
526 {
527 cout << " + would execute: " << cmd0 << endl;
528// cout << " + would execute: " << cmd1 << endl;
529// cout << " + would execute: " << cmd2 << endl;
530 return kTRUE;
531 }
532
533 cout << " + will execute: " << cmd0 << endl;
534// cout << " + will execute: " << cmd1 << endl;
535// cout << " + will execute: " << cmd2 << endl;
536
537 gSystem->Exec(cmd0);
538// gSystem->Exec(cmd1);
539// gSystem->Exec(cmd2);
540
541 return kTRUE;
542 }
543
544 Int_t CheckSequence(Long_t runstart, Long_t runstop, const TString &cond)
545 {
546 const char *fmt1 = "SELECT fRunID FROM RunInfo WHERE";
547 const char *fmt2 = "AND fExcludedFDAKEY=1 ORDER BY fRunID";
548
549 const TString query1 = Form("%s fSequenceID=%ld AND %s %s", fmt1, runstart, cond.Data(), fmt2);
550 const TString query2 = Form("%s fRunID BETWEEN %ld AND %ld AND %s %s", fmt1, runstart, runstop, cond.Data(), fmt2);
551
552 TSQLResult *res1 = Query(query1);
553 if (!res1)
554 return 2;
555
556 TSQLResult *res2 = Query(query2);
557 if (!res2)
558 {
559 delete res1;
560 return 2;
561 }
562
563 while (1)
564 {
565 TSQLRow *row1 = res1->Next();
566 TSQLRow *row2 = res2->Next();
567
568 if (!row1 && !row2)
569 return kTRUE;
570
571 if (!row1 || !row2)
572 return kFALSE;
573
574 if (atol((*row1)[0])!=atol((*row2)[0]))
575 return kFALSE;
576 }
577
578 return kFALSE;
579 }
580
581 Int_t CreateSequence(std::pair<Long_t,Long_t> &drs, Long_t runstart, Long_t runstop, const TString &cond)
582 {
583 cout << " * Creating Sequence " << runstart << "-" << runstop << " (drs=" << drs.first << "/" << drs.second << "):" << endl;
584
585 //----------------------------------------------------------------------------
586
587 TString query;
588 query = "SELECT fRunID FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
589 query += cond;
590 query += " AND fRunTypeKEY=4 AND fRunID BETWEEN ";
591 query += Form("%ld AND %ld", runstart, runstop);
592
593 TSQLResult *res = Query(query);
594 if (!res)
595 return kFALSE;
596
597 // Check if this is a DRS calibration (contains a G run)
598 if (res->GetRowCount()>0 && res->GetFieldCount()==1)
599 {
600 delete res;
601
602 // Get DRS file with pedestal (roi<1024)
603 query = "SELECT MAX(fRunID) FROM RunInfo WHERE ";
604 query += cond;
605 query += " AND fHasDrsFile=1 AND (fDrsStep IS NULL OR fDrsStep=2) AND fRunTypeKEY=2";
606 query += " AND fRunID BETWEEN ";
607 query += Form("%ld AND %ld", runstart, runstop);
608
609 res = Query(query);
610 if (!res)
611 return kFALSE;
612
613 if (res->GetRowCount()==0 || res->GetFieldCount()!=1)
614 return kFALSE;
615
616 TSQLRow *row = res->Next();
617
618 drs.first = runstart;
619 drs.second = (*row)[0] ? atol((*row)[0]) : -1;
620 }
621
622 delete res;
623
624 //----------------------------------------------------------------------------
625
626 query = "SELECT fSequenceID, fNight FROM RunInfo WHERE ";
627 query += cond;
628 query += " AND fRunID BETWEEN ";
629 query += Form("%ld AND %ld", runstart, runstop);
630 query += " AND fSequenceID>0";
631 query += " GROUP BY fSequenceID";
632
633 res = Query(query);
634 if (!res)
635 return -2;
636
637 const Int_t cnt = res->GetRowCount();
638
639 Int_t rc = kTRUE;
640 if (cnt==1)
641 {
642 TSQLRow *row=res->Next();
643 const Int_t check = CheckSequence(runstart, runstop, cond);
644 if (check==kTRUE)
645 {
646 cout << " - Identical sequence already existing." << endl;
647 delete res;
648 return -1; // kTRUE
649 }
650 if (check==2)
651 rc=2;
652 else
653 {
654 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
655 if (DeleteSequence(atoi((*row)[0]), atoi((*row)[1]))==2)
656 rc = 2;
657 }
658 }
659 else
660 {
661 TSQLRow *row=0;
662 while ((row=res->Next()))
663 {
664 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
665 if (DeleteSequence(atoi((*row)[0]), atoi((*row)[1]))==2)
666 rc = 2;
667 }
668 }
669
670 delete res;
671
672 if (rc==2)
673 return -2;
674
675 if (!InsertSequence(drs, runstart, runstop, cond))
676 return -2;
677
678 return kTRUE;
679 }
680
681 Bool_t ReadResources(const char *fname)
682 {
683 // Check for the section header
684 TPRegexp regexp("^\\[(RunTypes|Transition|Regexp):?[0-9 ]*\\]$");
685 // Check if section header contains a number
686 TPRegexp regnum("[0-9]");
687 // Check if section header contains the telescope number
688 TPRegexp regtel(Form("[^0-9]0*%d[^0-9]", fTelescopeNumber));
689
690 ifstream fin(fname);
691 if (!fin)
692 {
693 cout << "Cannot open file " << fname << ": ";
694 cout << strerror(errno) << endl;
695 return kFALSE;
696 }
697
698 Int_t section = 0;
699
700 TString key;
701 while (1)
702 {
703 TString txt;
704 txt.ReadLine(fin);
705 if (!fin)
706 break;
707
708 txt = txt.Strip(TString::kBoth);
709
710 if (txt[0]=='#' || txt.IsNull())
711 continue;
712
713 if (txt[0]=='[')
714 {
715 TString sec = txt(regexp);
716 if (!sec.IsNull())
717 {
718 // Skip sections with the wrong telescope number
719 // (If section contains numbers but not the tel num)
720 if (!sec(regnum).IsNull() && sec(regtel).IsNull())
721 continue;
722
723 // Check which section we are in
724 if (sec.BeginsWith("[RunTypes"))
725 section = 1;
726 if (sec.BeginsWith("[Transition"))
727 section = 2;
728 if (sec.BeginsWith("[Regexp"))
729 section = 3;
730
731 continue;
732 }
733
734 if (section!=3)
735 {
736 cout << "WARNING - Line starts with [ but we are not in the Regexp section." << endl;
737 cout << txt << endl;
738 continue;
739 }
740 }
741
742
743 TObjArray *arr = txt.Tokenize(" ");
744
745 if (arr->GetEntries()>0)
746 switch (section)
747 {
748 case 1:
749 // Create a llist of run-types
750 // drs-time T
751 break;
752 case 2:
753 {
754 TString key = arr->At(0)->GetName();
755 key.Prepend("f");
756 key.Append("KEY");
757
758 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
759 if (!list)
760 {
761 //cout << key << endl;
762 list = new CheckList;
763 fMap.Add(new TObjString(key), list);
764 }
765
766 if (arr->GetEntries()>1)
767 {
768 //cout << key << " ";
769 list->Add(new CheckMatch(*arr, 1));
770 }
771 }
772 break;
773 case 3:
774 fListRegexp.Add(new Rule(*arr));
775 break;
776 }
777
778 delete arr;
779 }
780
781 return kTRUE;
782 }
783/*
784 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
785 {
786 TObjArray names; // array with old names (including regexp)
787
788 // Add to array and expand the array if necessary
789 TSQLRow *row=0;
790 while ((row=res->Next()))
791 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
792
793 // Now a LUT is build which converts the keys for
794 // the names including the regexp into keys for
795 // the names excluding the regexp
796 TString elt(Form("ELT(RunData.f%sKEY+1", col));
797
798 // loop over all entries in the list
799 const Int_t n = names.GetSize();
800 for (int i=0; i<n; i++)
801 {
802 // For all entries which are not in the list
803 // write an undefined value into the LUT
804 TObject *o = names.UncheckedAt(i);
805 if (!o)
806 {
807 elt += ",0";
808 continue;
809 }
810
811 // Remove the regexp from the string which includes it
812 TString name = o->GetName();
813
814 TIter NextR(&regexp);
815 TObject *obj=0;
816 while ((obj=NextR()))
817 {
818 TPRegexp reg(obj->GetName());
819 const Ssiz_t pos = name.Index(reg, 0);
820 if (pos>0)
821 {
822 name.Remove(pos);
823 name += "-W";
824 break;
825 }
826 }
827
828 // Check if such a Key exists, if not insert it
829 const Int_t key = QueryKeyOfName(col, name);
830
831 // RESOLVE return code!!!
832 //if (key<0)
833 // return "";
834
835 // add index to the LUT
836 elt += Form(",%d", key);
837 }
838
839 // close LUT expression
840 // elt += ") AS Elt";
841 // elt += col;
842
843 elt += ") AS f";
844 elt += col;
845 elt += "KEY";
846
847 // return result
848 return elt;
849 }
850
851 TString GetELT(const char *col, TSQLResult *res, TString regexp)
852 {
853 TList list;
854 list.SetOwner();
855 list.Add(new TObjString(regexp));
856 return GetELT(col, res, list);
857 }
858
859 TString GetELTQuery(const char *col, const char *cond) const
860 {
861 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
862 "LEFT JOIN %s USING (f%sKEY) WHERE %s GROUP BY f%sName",
863 col, col, col, col, cond, col);
864 }
865
866 TString GetELTSource(const char *cond)
867 {
868 //query all sources observed in this night
869 TSQLResult *resx = Query(GetELTQuery("Source", cond));
870 if (!resx)
871 return "";
872
873 // In the case there is only a single source
874 // do not replace the source key by the ELT
875 if (resx->GetRowCount()==1)
876 return "fSourceKEY";
877
878 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
879 delete resx;
880
881 return elts;
882 }
883
884 TString GetELTProject(const char *cond)
885 {
886 //query all project names observed in this night
887 TSQLResult *resx = Query(GetELTQuery("Project", cond));
888 if (!resx)
889 return "";
890
891 // In the case there is only a single project
892 // do not replace the project key by the ELT
893 if (resx->GetRowCount()==1)
894 return "fProjectKEY";
895
896 TList regexp;
897 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
898 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
899
900 TString eltp2 = GetELT("Project", resx, regexp);
901 delete resx;
902 regexp.Delete();
903
904 return eltp2;
905 }
906*/
907 Bool_t HasAtLeastOne(TString src, TString chk) const
908 {
909 src.ToLower();
910 chk.ToLower();
911
912 for (int i=0; i<chk.Length(); i++)
913 if (src.First(chk[i])<0)
914 return kFALSE;
915
916 return kTRUE;
917 }
918
919 TString PrepareString(TSQLResult &res, TArrayL &runs, TArrayL &nights)
920 {
921 // Number of result rows
922 const Int_t rows = res.GetRowCount();
923
924 runs.Set(rows); // initialize size of array for run numbers
925 nights.Set(rows); // initialize size of array for nights
926
927 TArrayD start(rows); // instantiate array for start times
928 TArrayD stop(rows); // instantiate array for stop times
929
930 TString str; // result string
931
932 Int_t idx=0;
933 TSQLRow *row=0;
934 while ((row=res.Next()))
935 {
936 runs[idx] = atol((*row)[0]); // run number
937 nights[idx] = atol((*row)[4]); // night
938
939 const TString tstart = ((*row)[2]); // start time
940 const TString tstop = ((*row)[3]); // stop time
941
942 start[idx] = MTime(tstart).GetMjd(); // convert to double
943 stop[idx] = MTime(tstop).GetMjd(); // convert to double
944
945 // This is a workaround for broken start-times
946 if (tstart=="0000-00-00 00:00:00")
947 start[idx] = stop[idx];
948
949 // Add a run-type character for this run to the string
950 str += RunType((*row)[1]);
951
952 // Increase index
953 idx++;
954 }
955/*
956 // Now the P- and D- runs are classified by the time-distance
957 // to the next or previous C-Run
958 Double_t lastc = -1;
959 Double_t nextc = -1;
960 for (int i=0; i<str.Length(); i++)
961 {
962 if (str[i]=='C')
963 {
964 // Remember stop time of C-Run as time of last C-run
965 lastc = stop[i];
966
967 // Calculate start time of next C-Run
968 const TString residual = str(i+1, str.Length());
969 const Int_t pos = residual.First('C');
970
971 // No C-Run found anymore. Finished...
972 if (pos<0)
973 break;
974
975 // Remember start time of C-Run as time of next C-run
976 nextc = start[i+1+pos];
977 continue;
978 }
979
980 // Check whether the identifying character should
981 // be converted to lower case
982 if (start[i]-lastc>nextc-stop[i])
983 str[i] = tolower(str[i]);
984 }
985*/
986 //cout << str << endl;
987 return str;
988 }
989
990 void PrintResidual(Long_t runstart, Long_t runstop, TString residual, const char *descr)
991 {
992 residual.ToLower();
993
994 // Count number of unsequences "characters"
995 const Int_t numd = residual.CountChar('d');
996 const Int_t nump = residual.CountChar('p');
997 const Int_t numb = residual.CountChar('b');
998 const Int_t numg = residual.CountChar('g');
999 const Int_t numt = residual.CountChar('t');
1000 const Int_t numc = residual.CountChar('c');
1001
1002 // Print some information to the output steram
1003 if (numd+nump+numb+numg+numt+numc==0)
1004 return;
1005
1006 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
1007 if (numd>0)
1008 cout << " (DATA!)";
1009/*
1010 if (nump==0 || numc==0 || numd==0)
1011 cout << " Missing";
1012 if (numd==0)
1013 cout << " D";
1014 if (nump==0)
1015 cout << " P";
1016 if (numc==0)
1017 cout << " C";
1018*/
1019 cout << endl;
1020 }
1021
1022 Int_t SplitBlock(Long_t runstart, Long_t runstop, const TString &cond)
1023 {
1024 // Request data necessary to split block into sequences
1025 /*const*/
1026 TString query =
1027 "SELECT fRunID, fRunTypeKEY, fRunStart, fRunStop, fNight*1000 "
1028 " FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
1029 query += Form("fRunID BETWEEN %ld AND %ld AND "
1030 /*" (fRunTypeKEY BETWEEN 2 AND 4)"*/, runstart, runstop);
1031
1032 query += cond;
1033 query += " ORDER BY fRunID";
1034
1035
1036 // Send query
1037 TSQLResult *res = Query(query);
1038 if (!res)
1039 return 2;
1040
1041 // Get String containing the sequence of P-,C- and D-Runs
1042 // and an array with the corresponding run-numbers
1043 TArrayL runs;
1044 TArrayL nights;
1045 const TString str = PrepareString(*res, runs, nights);
1046
1047 delete res;
1048
1049 // Check if the prepared string at least contains one run of each type
1050 // ??????????????????????????????????
1051 /*
1052 if (!HasAtLeastOne(str, "PCD"))
1053 {
1054 PrintResidual(runstart, runstop, str, "Block");
1055 return kTRUE;
1056 }*/
1057 // ??????????????????????????????????
1058
1059 // Start the sequence building
1060 // ---------------------------
1061
1062 Int_t pos = 0; // Position in the string
1063 std::map<Long_t, Long_t> map; // Map for resulting sequences
1064 TString residual; // Unsequences "characters"
1065
1066 // Step through the string one character by one
1067 cout << " ";
1068 while (pos<str.Length())
1069 {
1070 Bool_t found = kFALSE;
1071
1072 // Loop over the predefined regular expressions
1073 TIter Next(&fListRegexp);
1074 Rule *obj = 0;
1075 while ((obj=(Rule*)Next()))
1076 {
1077 // Check if regular expressions matchs
1078 const Ssiz_t len = obj->Match(str, pos, nights[pos]);
1079 if (len>0)
1080 {
1081 // In case of match get the substring which
1082 // defines the sequence and print it
1083 TString sub = str(pos, len);
1084 sub.ToUpper();
1085 cout << runs[pos]<<":"<< sub << "|";
1086
1087 // Add the first and last run of the sequence to the map
1088 //map.Add(runs[pos], runs[pos+len-1]);
1089 map[runs[pos]] = runs[pos+len-1];
1090
1091 // A sequence was found...
1092 found = kTRUE;
1093
1094 // step forward in the string by the length of the sequence
1095 pos += len;
1096 break;
1097 }
1098 }
1099
1100 // If a sequence was found go on...
1101 if (found)
1102 continue;
1103
1104 // print unsequenced characters
1105 cout << (char)tolower(str[pos]);
1106
1107 // Count the number of "characters" not sequenced
1108 residual += str[pos];
1109
1110 // step one character forward
1111 pos++;
1112 }
1113 cout << endl;
1114
1115 PrintResidual(runstart, runstop, residual, "Runs ");
1116
1117 // Create all sequences which were previously found
1118 std::pair<Long_t,Long_t> drs(-1, -1);
1119 for (std::map<Long_t,Long_t>::const_iterator it=map.begin();
1120 it!=map.end(); it++)
1121 if (CreateSequence(drs, it->first, it->second, cond)==2)
1122 return 2;
1123 /*
1124 Long64_t first, last;
1125 TExMapIter iter(&map);
1126 while (iter.Next(first, last))
1127 if (CreateSequence(first, last, cond)==2)
1128 return 2;
1129 */
1130 return kTRUE;
1131 }
1132
1133 Char_t RunType(Int_t n) const
1134 {
1135 switch (n)
1136 {
1137 case 1: return 'D';
1138 case 2: return 'P'; // Ped1024
1139 case 3: return 'B'; // Ped 300
1140 case 4: return 'G'; // drs-gain
1141 case 5: return 'T'; // drs-time
1142 case 6: return 'C'; // light-pulser-ext
1143 case 11: return 'M'; // ped-and-lp-ext
1144 case 12: return 'U'; // drs-time-upshifted
1145 }
1146 return '-';
1147 }
1148 Char_t RunType(const char *str) const
1149 {
1150 return RunType(atoi(str));
1151 }
1152
1153 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks, const TString &cond)
1154 {
1155 // col key content
1156 // -----------------------------
1157 // 0 - runnumber
1158 // 1 0 RunTypeKEY
1159 // 2 1 source
1160 // 3 2 project
1161 // . .
1162 // . .
1163 // . . from transition.txt
1164 //
1165
1166 //build blocks of runs, which have the same values
1167 //for each block the first and the last run are stored in a TExMap
1168 //the values are checked with the help of an array of TStrings
1169 Long_t runstart = -1;
1170 Long_t runstop = -1;
1171
1172 const UInt_t ncols = res->GetFieldCount();
1173
1174 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
1175
1176 // Loop over runs
1177 TSQLRow *row=0;
1178 while ((row=res->Next()))
1179 {
1180 // This is the runnumber of the first run in the new block
1181 if (runstart<0)
1182 runstart=atol((*row)[0]);
1183
1184 // Check which transitions might be allowed for this run and
1185 // which are not. Check also which attributes should be ignored
1186 // for this run. The information about it is stored in the map.
1187 Int_t rc[ncols];
1188 for (UInt_t i=2; i<ncols; i++)
1189 rc[i] = CheckTransition(*res, keys, *row, i);
1190
1191 for (UInt_t i=2; i<ncols; i++)
1192 {
1193 switch (rc[i])
1194 {
1195 case kTRUE: // Transition allowed, switch to new attribute
1196 keys[i] = (*row)[i];
1197 continue;
1198
1199 case -1: // Ignore attribute, continue with next one
1200 //if (keys[i] == (*row)[i])
1201 // break; // Only ignore it if it is really different
1202 continue;
1203
1204 case kFALSE: // No decision (nothing matching found in map)
1205 break; // go on checking
1206
1207 }
1208
1209 // If past-attribute is not yet initialized, do it now.
1210 if (keys[i].IsNull())
1211 keys[i] = (*row)[i];
1212
1213 // Check whether key has changed for this run
1214 // (this condition is never true for the first run)
1215 // This condition in only reached if keys[i] was initialized
1216 if (keys[i] == (*row)[i])
1217 continue;
1218
1219 // Found one block with unique keys, fill values into TExMap
1220 // (except if this is the first run processed)
1221 cout << endl;
1222 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1223 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1224 if (SplitBlock(runstart, runstop, cond)==2)
1225 return 2;
1226 cout << " - Transition from " << RunType(keys[1]) << ":";
1227 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1228 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1229 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1230
1231 // This is already the first run of the new block
1232 runstart=atol((*row)[0]);
1233
1234 // These are the keys corresponding to the first run
1235 // in the new block. All following runs should have
1236 // identical keys. Do not set the attribute if for
1237 // this attribute the transition check evaluated
1238 // to "ignore".
1239 for (UInt_t j=2; j<ncols; j++)
1240 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1241
1242 break;
1243 }
1244
1245 keys[1] = (*row)[1]; // Remember run type for next run
1246
1247 // This is the new runnumber of the last run in this block
1248 runstop=atol((*row)[0]);
1249 }
1250
1251 // If runstart==-1 a possible block would contain a single run...
1252 if (runstart>0 && runstart!=runstop)
1253 {
1254 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1255 //fill values into TExMap (last value)
1256 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1257 if (SplitBlock(runstart, runstop, cond)==2)
1258 return 2;
1259 }
1260 cout << "Done." << endl << endl;
1261
1262 return kTRUE;
1263 }
1264
1265public:
1266 SequenceBuild(Int_t tel=1, const char *rc="sql.rc") : MSQLMagic(rc), fTelescopeNumber(tel)
1267 {
1268 fListRegexp.SetOwner();
1269
1270 // FIXME: THIS IS NOT YET HANDLED
1271 if (ReadResources("fact/resources/sequences.rc"))
1272 return;
1273 }
1274
1275 SequenceBuild(TEnv &env, Int_t tel=1) : MSQLMagic(env), fTelescopeNumber(tel)
1276 {
1277 fListRegexp.SetOwner();
1278
1279 // FIXME: THIS IS NOT YET HANDLED
1280 if (ReadResources("fact/resources/sequences.rc"))
1281 return;
1282 }
1283 ~SequenceBuild()
1284 {
1285 fMap.DeleteAll();
1286 }
1287
1288 void SetPathRawData(const char *path) { fPathRawData=path; }
1289 void SetPathSequences(const char *path) { fPathSequences=path; }
1290
1291 int Build(TString day)
1292 {
1293 cout << endl;
1294 cout << "Night of sunset at " << day << ":" << endl;
1295
1296 const TString cond = Form("fNight=%s ", day.Data());
1297
1298 // Form("fRunNumber=%s "
1299 // "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 "
1300 // "AND fTelescopeNumber=%d ",
1301 // day.Data(), fTelescopeNumber);
1302/*
1303 //query all sources observed in this night
1304 const TString elts = GetELTSource(cond);
1305 if (elts.IsNull())
1306 return 2;
1307
1308 //query all project names observed in this night
1309 const TString eltp2 = GetELTProject(cond);
1310 if (elts.IsNull())
1311 return 2;
1312*/
1313 // Setup query to get all values from the database,
1314 // that are relevant for building sequences
1315 TString query("SELECT fRunID, fRunTypeKEY ");
1316/*
1317 query += elts;
1318 query += ", ";
1319 query += eltp2;
1320*/
1321 // Now add all entries from the transition table to the query
1322 TIter NextPair(&fMap);
1323 TObject *mapkey = 0;
1324 while ((mapkey=(TPair*)NextPair()))
1325 if (!query.Contains(mapkey->GetName()))
1326 {
1327 query += ", ";
1328 query += mapkey->GetName();
1329 }
1330 query += " FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
1331 query += cond;
1332 query += " ORDER BY fRunID";
1333
1334 TSQLResult *res = Query(query);
1335 if (!res)
1336 return 2;
1337
1338 TExMap blocks;
1339 const Int_t rc = BuildBlocks(res, blocks, cond);
1340 delete res;
1341
1342 return rc;
1343 }
1344
1345 Int_t Build()
1346 {
1347 //get all dates from the database
1348 TSQLResult *res = Query("SELECT fNight FROM RunInfo GROUP BY fNight ORDER BY fNight DESC");
1349 if (!res)
1350 return 2;
1351
1352 //execute buildsequenceentries for all dates
1353 TSQLRow *row=0;
1354 while ((row=res->Next()))
1355 Build((*row)[0]);
1356
1357 delete res;
1358
1359 return 1;
1360 }
1361 ClassDef(SequenceBuild, 0)
1362};
1363
1364ClassImp(SequenceBuild);
1365
1366int buildseqentries(TString day, TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1367{
1368 SequenceBuild serv(tel, "sql.rc");
1369 if (!serv.IsConnected())
1370 {
1371 cout << "ERROR - Connection to database failed." << endl;
1372 return 0;
1373 }
1374
1375 cout << "buildseqentries" << endl;
1376 cout << "---------------" << endl;
1377 cout << endl;
1378 cout << "Connected to " << serv.GetName() << endl;
1379 if (!datapath.IsNull())
1380 cout << "DataPath: " << datapath << endl;
1381 if (!sequpath.IsNull())
1382 cout << "SeqPath: " << sequpath << endl;
1383 cout << "Day: " << day << endl;
1384 cout << "Telescope: " << tel << endl;
1385 cout << endl;
1386
1387 serv.SetIsDummy(dummy);
1388 serv.SetPathRawData(datapath);
1389 serv.SetPathSequences(sequpath);
1390 return serv.Build(day);
1391}
1392
1393//
1394// Build Sequences for all Nights
1395//
1396int buildseqentries(TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1397{
1398 SequenceBuild serv(tel, "sql.rc");
1399 if (!serv.IsConnected())
1400 {
1401 cout << "ERROR - Connection to database failed." << endl;
1402 return 0;
1403 }
1404
1405 cout << "buildseqentries" << endl;
1406 cout << "---------------" << endl;
1407 cout << endl;
1408 cout << "Connected to " << serv.GetName() << endl;
1409 cout << "DataPath: " << datapath << endl;
1410 cout << "SeqPath: " << sequpath << endl;
1411 cout << "Telescope: " << tel << endl;
1412 cout << endl;
1413
1414 serv.SetIsDummy(dummy);
1415 serv.SetPathRawData(datapath);
1416 serv.SetPathSequences(sequpath);
1417 return serv.Build();
1418}
1419
1420int buildseqentries(Int_t tel=1, Bool_t dummy=kTRUE)
1421{
1422 return buildseqentries("", "", tel, dummy);
1423}
1424
1425int buildseqentries(TString day, Int_t tel=1, Bool_t dummy=kTRUE)
1426{
1427 return buildseqentries(day, "", "", tel, dummy);
1428}
Note: See TracBrowser for help on using the repository browser.