source: branches/MarsISDCBranchBasedOn17887/fact/processing/buildseqentries.C

Last change on this file was 15237, checked in by Daniela Dorner, 12 years ago
adapted path for setup file for sequence definition
File size: 44.2 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82
83#include <map>
84#include <utility>
85
86#include <errno.h>
87
88#include <TSQLRow.h>
89#include <TSQLResult.h>
90
91#include <TEnv.h>
92#include <TMap.h>
93#include <TMath.h>
94#include <TExMap.h>
95#include <TArrayI.h>
96#include <TArrayL.h>
97#include <TArrayD.h>
98#include <TPRegexp.h>
99#include <TSystem.h>
100#include <TObjString.h>
101#include <TObjArray.h>
102
103#include "MTime.h"
104#include "MSequence.h"
105
106#include "MSQLMagic.h"
107
108using namespace std;
109
110 /*static*/ ULong_t GetId(const TString str)
111 {
112 const Ssiz_t dot = str.First('.');
113
114 const UInt_t run = str.Atoi();
115 const UInt_t sub = dot<0 ? 0 : atoi(str.Data()+dot+1);
116
117 return ULong_t(run)*1000+sub;
118 }
119
120
121class Rule : public TObject
122{
123private:
124 TPRegexp fRegexp;
125
126 ULong_t fMin;
127 ULong_t fMax;
128
129public:
130 Rule(TObjArray &arr) :
131 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
132 fMin(0), fMax((ULong_t)-1)
133 {
134 if (arr.GetEntries()>1)
135 fMin = GetId(arr[1]->GetName());
136 if (arr.GetEntries()>2)
137 fMax = GetId(arr[2]->GetName());
138 }
139
140 Ssiz_t Match(const TString &str, Int_t idx, Long_t run=-1)
141 {
142 if (!IsValid(run))
143 return 0;
144
145 TString mods;
146 TArrayI pos;
147 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
148
149 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
150 }
151
152 Bool_t IsValid(ULong_t run) const { return run>=fMin && run<=fMax; }
153 ClassDef(Rule, 0)
154};
155ClassImp(Rule);
156
157class CheckMatch : public TObject
158{
159private:
160 TPRegexp fRunType1;
161 TPRegexp fRunType2;
162
163 TPRegexp fRegexp1;
164 TPRegexp fRegexp2;
165
166 ULong_t fMin;
167 ULong_t fMax;
168
169 void Init(const TObjArray &arr, Int_t idx=0)
170 {
171 const Int_t n = arr.GetEntries();
172
173 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
174
175 for (int i=0; i<n-idx; i++)
176 {
177 //cout << arr[i+idx]->GetName() << " ";
178
179 TString str(arr[i+idx]->GetName());
180 if (str=="*")
181 str = ".*";
182
183 switch (isminus && i>1 ? i+1 : i)
184 {
185 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
186 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
187 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
188 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
189 case 4: fMin = GetId(str); break;
190 case 5: fMax = GetId(str); break;
191 }
192 }
193 }
194
195public:
196 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
197
198 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
199 {
200 TObjArray *arr = txt.Tokenize(" ");
201 Init(*arr);
202 delete arr;
203 }
204
205 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
206 {
207 Init(arr, idx);
208 }
209
210 CheckMatch(const char *from, const char *to, Long_t min=0, Long_t max=-1)
211 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
212 {
213 }
214
215 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, ULong_t run=0)
216 {
217 if (run>0)
218 {
219 if (run<fMin || run>fMax)
220 return kFALSE;
221 }
222
223 const TString test("X-"); // FIXME:STUPID!
224
225 //if (test.Index(fRunType2)==1)
226 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
227
228 if (test.Index(fRunType1,0)==1)
229 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
230
231 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
232 }
233 ClassDef(CheckMatch,0)
234};
235ClassImp(CheckMatch);
236
237class CheckList : public TList
238{
239public:
240 CheckList() { SetOwner(); }
241 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Long_t run=-1) const
242 {
243 TIter Next(this);
244
245 CheckMatch *check = 0;
246
247 while ((check=(CheckMatch*)Next()))
248 {
249 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
250 if (rc)
251 return rc;
252 }
253
254 return kFALSE;
255 }
256 ClassDef(CheckList,0)
257};
258ClassImp(CheckList);
259
260class SequenceBuild : public MSQLMagic
261{
262private:
263 TString fPathRawData;
264 TString fPathSequences;
265
266 Int_t fTelescopeNumber;
267
268 TMap fMap;
269 TList fListRegexp;
270
271 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
272 {
273 // Get the name of the column from result table
274 const TString key = res.GetFieldName(i);
275
276 // Get the list with the allowed attributed from the map
277 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
278 if (!list)
279 return kFALSE;
280
281 // Check whether the current run (row[0]) with the current attribute
282 // (row[i]) and the current run-type (row[1]) matches the attribute
283 // of the last run (keys[i] with run-type row[i])
284 return list->Matches(keys[1], row[1], keys[i], row[i], atol(row[0]));
285 }
286
287 Bool_t InsertSequence(std::pair<Long_t,Long_t> drs, Long_t from, Long_t to, const TString &cond)
288 {
289 //cout << " - Inserting Sequence into database." << endl;
290
291 TString query;
292 query = "SELECT fNight, STR_TO_DATE(fNight,'%Y%m%d') ";
293 query += " FROM RunInfo WHERE ";
294 query += cond;
295 query += Form(" AND fRunID=%ld", from);
296
297 TSQLResult *res = Query(query);
298 if (!res)
299 return kFALSE;
300
301 if (res->GetFieldCount()!=2 || res->GetRowCount()!=1)
302 {
303 cout << "ERROR - Wrong result from query: " << query << endl;
304 return kFALSE;
305 }
306
307 TSQLRow *row = res->Next();
308
309 UInt_t run = atoi((*row)[0]);
310 TString day = (*row)[1];
311
312 delete res;
313
314 query = "SELECT fRunID, fRunTypeKEY, fHasDrsFile, fDrsStep ";
315 query += " FROM RunInfo WHERE ";
316 query += cond;
317 query += Form(" AND fRunID BETWEEN %ld AND %ld ORDER BY fRunID", from, to);
318
319 if (!(res = Query(query)))
320 return kFALSE;
321
322 if (res->GetFieldCount()!=4)
323 {
324 cout << "ERROR - Wrong result from query: " << query << endl;
325 return kFALSE;
326 }
327
328 MSequence seq("sequence.def", "", (run%1000000)*1000+from);
329 seq.SetNight(day);
330
331 if (drs.first!=from)
332 {
333 seq.SetDrsSequence(drs.first);
334 if (drs.second!=-1)
335 seq.AddFile(drs.second, -1, MSequence::kDrsFile);
336 }
337
338 while ((row=res->Next()))
339 {
340 UInt_t num = atol((*row)[0]);
341 UInt_t key = atol((*row)[1]);
342
343 switch (key)
344 {
345 case 1: seq.AddFile(num, -1, MSequence::kDat); break;
346 case 2: seq.AddFile(num, -1, MSequence::kPed); break;
347 case 3: seq.AddFile(num, -1, MSequence::kDrsRun); break;
348 case 4: seq.AddFile(num, -1, MSequence::kCal); break;
349 case 5: seq.AddFile(num, -1, MSequence::kDat); break;
350 case 6: seq.AddFile(num, -1, MSequence::kCal); break;
351 }
352
353 if (atoi((*row)[2])==1)
354 if (!(*row)[3] || atoi((*row)[3])==2)
355 seq.AddFile(num, -1, MSequence::kDrsFile);
356 }
357
358 const TString path =
359 Form("%s/%04d/%02d/%02d", fPathSequences.Data(),
360 run/10000, (run/100)%100, run%100);
361
362 const TString fname =
363 Form("%s/%d_%03ld.%s", path.Data(), run, from,
364 drs.first==from?"drs.seq":"seq");
365
366 if (IsDummy())
367 {
368 cout << "----- " << fname << " -----" << endl;
369 seq.Print();
370 cout << "^^^^^ " << fname << " ^^^^^" << endl;
371 }
372 else
373 {
374 if (gSystem->AccessPathName(path, kFileExists))
375 if (gSystem->mkdir(path, kTRUE)<0)
376 cout << "ERROR - Could not create " << path << endl;
377
378 if (!gSystem->AccessPathName(path, kFileExists))
379 seq.WriteFile(fname);
380 }
381
382 delete res;
383
384 if (drs.first==from)
385 return kTRUE;
386
387 // ========== Request number of events ==========
388
389 // Can be replaced by
390
391 const TString runtime =
392 "SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))), ";
393
394 const TString where = Form("(fRunID BETWEEN %ld AND %ld) AND fExcludedFDAKEY=1 AND %s",
395 from, to, cond.Data());
396
397 query = "SELECT SUM(fNumEvents), ";
398 query += runtime;
399 query += " MIN(fZenithDistanceMin), MAX(fZenithDistanceMax), ";
400 query += " MIN(fAzimuthMin), MAX(fAzimuthMax), ";
401 query += " MIN(fRunStart), MAX(fRunStop) ";
402 query += " FROM RunInfo WHERE ";
403 query += where;
404 query += " AND fRunTypeKEY=1";// GROUP BY fNight";
405
406 res = Query(query);
407 if (!res)
408 return kFALSE;
409
410 row = res->Next();
411 if (!row || res->GetFieldCount()!=8)
412 {
413 cout << "ERROR - Wrong result from query: " << query << endl;
414 return kFALSE;
415 }
416
417 const TString nevts = (*row)[0];
418 const TString secs = (*row)[1];
419 const TString zdmin = (*row)[2];
420 const TString zdmax = (*row)[3];
421 const TString azmin = (*row)[4];
422 const TString azmax = (*row)[5];
423 const TString start = (*row)[6];
424 const TString stop = (*row)[7];
425
426 delete res;
427/*
428 const TString elts = GetELTSource(where);
429 if (elts.IsNull())
430 return kFALSE;
431
432 const TString eltp = GetELTProject(where);
433 if (eltp.IsNull())
434 return kFALSE;
435
436 // ========== Request data of sequence ==========
437 query = "SELECT ";
438
439 query += elts;
440 query += ", ";
441 query += eltp;
442
443 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
444 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
445 " fTriggerDelayTableKEY, fObservationModeKEY, fSumTriggerFlagKEY "
446 " FROM RunData WHERE fRunTypeKEY=2 AND ";
447 query += where;
448 query += " LIMIT 1";
449
450 res = Query(query);
451 if (!res)
452 return kFALSE;
453
454 row = res->Next();
455 if (!row || res->GetFieldCount()!=9)
456 {
457 cout << "ERROR - No result from query: " << query << endl;
458 return kFALSE;
459 }
460 */
461
462 const TString set1 = Form("fSequenceID=%ld", from);
463 const TString set2 = set1 + Form(", fNight=%d", run);
464 const TString set3 = set2 + Form(", fStartTime=Now(), fStopTime=Now(), fProcessingSiteKEY=4, fAvailable=Now() ");
465
466 TString query1;
467 query1 += set2+",";
468 query1 += Form(" fNumEvents=%s,", nevts.Data());
469 query1 += Form(" fSequenceDuration=%s,", secs.Data());
470 query1 += Form(" fSequenceStart=\"%s\",", start.Data());
471 query1 += Form(" fSequenceStop=\"%s\",", stop.Data());
472 query1 += Form(" fZenithDistanceMin=%s,", zdmin.IsNull() ? "NULL" : zdmin.Data());
473 query1 += Form(" fZenithDistanceMax=%s,", zdmax.IsNull() ? "NULL" : zdmax.Data());
474 query1 += Form(" fAzimuthMin=%s,", azmin.IsNull() ? "NULL" : azmin.Data());
475 query1 += Form(" fAzimuthMax=%s", azmax.IsNull() ? "NULL" : azmax.Data());
476
477 if (!Insert("SequenceInfo", query1))
478 {
479 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
480 return kFALSE;
481 }
482
483 if (!Insert("SequenceFileAvailISDCStatus", set3))
484 {
485 cout << "ERROR - Could not insert Sequence into SequenceFileAvailISDCStatus." << endl;
486 return kFALSE;
487 }
488
489 if (!Update("RunInfo", set1, where))
490 {
491 cout << "ERROR - Could not update RunData." << endl;
492 return kFALSE;
493 }
494
495 return kTRUE;
496 }
497
498 Bool_t DeleteSequence(Int_t sequ, UInt_t night)
499 {
500 if (fPathRawData.IsNull() || fPathSequences.IsNull())
501 {
502 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
503 return kTRUE;
504 }
505
506 //queries to delete information from the database
507 const TString cond(Form("fSequenceID=%d AND fNight=%d", sequ, night));
508
509 const TString path(Form("%04d/%02d/%02d", night/10000, (night/100)%100, night%100));
510
511 //commands to delete files from the disk
512 const TString cmd0(Form("rm -f %s/%s/%08d_%03d*seq", fPathSequences.Data(), path.Data(), night, sequ));
513
514// const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
515// const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
516
517 if (!Delete("SequenceInfo", cond))
518 return 2;
519
520 if (!Update("RunInfo", "fSequenceID=NULL", cond))
521 return 2;
522
523 if (IsDummy())
524 {
525 cout << " + would execute: " << cmd0 << endl;
526// cout << " + would execute: " << cmd1 << endl;
527// cout << " + would execute: " << cmd2 << endl;
528 return kTRUE;
529 }
530
531 cout << " + will execute: " << cmd0 << endl;
532// cout << " + will execute: " << cmd1 << endl;
533// cout << " + will execute: " << cmd2 << endl;
534
535 gSystem->Exec(cmd0);
536// gSystem->Exec(cmd1);
537// gSystem->Exec(cmd2);
538
539 return kTRUE;
540 }
541
542 Int_t CheckSequence(Long_t runstart, Long_t runstop, const TString &cond)
543 {
544 const char *fmt1 = "SELECT fRunID FROM RunInfo WHERE";
545 const char *fmt2 = "AND fExcludedFDAKEY=1 ORDER BY fRunID";
546
547 const TString query1 = Form("%s fSequenceID=%ld AND %s %s", fmt1, runstart, cond.Data(), fmt2);
548 const TString query2 = Form("%s fRunID BETWEEN %ld AND %ld AND %s %s", fmt1, runstart, runstop, cond.Data(), fmt2);
549
550 TSQLResult *res1 = Query(query1);
551 if (!res1)
552 return 2;
553
554 TSQLResult *res2 = Query(query2);
555 if (!res2)
556 {
557 delete res1;
558 return 2;
559 }
560
561 while (1)
562 {
563 TSQLRow *row1 = res1->Next();
564 TSQLRow *row2 = res2->Next();
565
566 if (!row1 && !row2)
567 return kTRUE;
568
569 if (!row1 || !row2)
570 return kFALSE;
571
572 if (atol((*row1)[0])!=atol((*row2)[0]))
573 return kFALSE;
574 }
575
576 return kFALSE;
577 }
578
579 Int_t CreateSequence(std::pair<Long_t,Long_t> &drs, Long_t runstart, Long_t runstop, const TString &cond)
580 {
581 cout << " * Creating Sequence " << runstart << "-" << runstop << " (drs=" << drs.first << "/" << drs.second << "):" << endl;
582
583 //----------------------------------------------------------------------------
584
585 TString query;
586 query = "SELECT fRunID FROM runinfo WHERE fExcludedFDAKEY=1 AND ";
587 query += cond;
588 query += " AND fRunTypeKEY=4 AND fRunID BETWEEN ";
589 query += Form("%ld AND %ld", runstart, runstop);
590
591 TSQLResult *res = Query(query);
592 if (!res)
593 return kFALSE;
594
595 // Check if this is a DRS calibration (contains a G run)
596 if (res->GetRowCount()>0 && res->GetFieldCount()==1)
597 {
598 delete res;
599
600 // Get DRS file with pedestal (roi<1024)
601 query = "SELECT MAX(fRunID) FROM runinfo WHERE ";
602 query += cond;
603 query += " AND fHasDrsFile=1 AND (fDrsStep IS NULL OR fDrsStep=2) AND fRunTypeKEY=2";
604 query += " AND fRunID BETWEEN ";
605 query += Form("%ld AND %ld", runstart, runstop);
606
607 res = Query(query);
608 if (!res)
609 return kFALSE;
610
611 if (res->GetRowCount()==0 || res->GetFieldCount()!=1)
612 return kFALSE;
613
614 TSQLRow *row = res->Next();
615
616 drs.first = runstart;
617 drs.second = (*row)[0] ? atol((*row)[0]) : -1;
618 }
619
620 delete res;
621
622 //----------------------------------------------------------------------------
623
624 query = "SELECT fSequenceID, fNight FROM RunInfo WHERE ";
625 query += cond;
626 query += " AND fRunID BETWEEN ";
627 query += Form("%ld AND %ld", runstart, runstop);
628 query += " AND fSequenceID>0";
629 query += " GROUP BY fSequenceID";
630
631 res = Query(query);
632 if (!res)
633 return -2;
634
635 const Int_t cnt = res->GetRowCount();
636
637 Int_t rc = kTRUE;
638 if (cnt==1)
639 {
640 TSQLRow *row=res->Next();
641 const Int_t check = CheckSequence(runstart, runstop, cond);
642 if (check==kTRUE)
643 {
644 cout << " - Identical sequence already existing." << endl;
645 delete res;
646 return -1; // kTRUE
647 }
648 if (check==2)
649 rc=2;
650 else
651 {
652 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
653 if (DeleteSequence(atoi((*row)[0]), atoi((*row)[1]))==2)
654 rc = 2;
655 }
656 }
657 else
658 {
659 TSQLRow *row=0;
660 while ((row=res->Next()))
661 {
662 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
663 if (DeleteSequence(atoi((*row)[0]), atoi((*row)[1]))==2)
664 rc = 2;
665 }
666 }
667
668 delete res;
669
670 if (rc==2)
671 return -2;
672
673 if (!InsertSequence(drs, runstart, runstop, cond))
674 return -2;
675
676 return kTRUE;
677 }
678
679 Bool_t ReadResources(const char *fname)
680 {
681 // Check for the section header
682 TPRegexp regexp("^\\[(RunTypes|Transition|Regexp):?[0-9 ]*\\]$");
683 // Check if section header contains a number
684 TPRegexp regnum("[0-9]");
685 // Check if section header contains the telescope number
686 TPRegexp regtel(Form("[^0-9]0*%d[^0-9]", fTelescopeNumber));
687
688 ifstream fin(fname);
689 if (!fin)
690 {
691 cout << "Cannot open file " << fname << ": ";
692 cout << strerror(errno) << endl;
693 return kFALSE;
694 }
695
696 Int_t section = 0;
697
698 TString key;
699 while (1)
700 {
701 TString txt;
702 txt.ReadLine(fin);
703 if (!fin)
704 break;
705
706 txt = txt.Strip(TString::kBoth);
707
708 if (txt[0]=='#' || txt.IsNull())
709 continue;
710
711 if (txt[0]=='[')
712 {
713 TString sec = txt(regexp);
714 if (!sec.IsNull())
715 {
716 // Skip sections with the wrong telescope number
717 // (If section contains numbers but not the tel num)
718 if (!sec(regnum).IsNull() && sec(regtel).IsNull())
719 continue;
720
721 // Check which section we are in
722 if (sec.BeginsWith("[RunTypes"))
723 section = 1;
724 if (sec.BeginsWith("[Transition"))
725 section = 2;
726 if (sec.BeginsWith("[Regexp"))
727 section = 3;
728
729 continue;
730 }
731
732 if (section!=3)
733 {
734 cout << "WARNING - Line starts with [ but we are not in the Regexp section." << endl;
735 cout << txt << endl;
736 continue;
737 }
738 }
739
740
741 TObjArray *arr = txt.Tokenize(" ");
742
743 if (arr->GetEntries()>0)
744 switch (section)
745 {
746 case 1:
747 // Create a llist of run-types
748 // drs-time T
749 break;
750 case 2:
751 {
752 TString key = arr->At(0)->GetName();
753 key.Prepend("f");
754 key.Append("KEY");
755
756 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
757 if (!list)
758 {
759 //cout << key << endl;
760 list = new CheckList;
761 fMap.Add(new TObjString(key), list);
762 }
763
764 if (arr->GetEntries()>1)
765 {
766 //cout << key << " ";
767 list->Add(new CheckMatch(*arr, 1));
768 }
769 }
770 break;
771 case 3:
772 fListRegexp.Add(new Rule(*arr));
773 break;
774 }
775
776 delete arr;
777 }
778
779 return kTRUE;
780 }
781/*
782 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
783 {
784 TObjArray names; // array with old names (including regexp)
785
786 // Add to array and expand the array if necessary
787 TSQLRow *row=0;
788 while ((row=res->Next()))
789 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
790
791 // Now a LUT is build which converts the keys for
792 // the names including the regexp into keys for
793 // the names excluding the regexp
794 TString elt(Form("ELT(RunData.f%sKEY+1", col));
795
796 // loop over all entries in the list
797 const Int_t n = names.GetSize();
798 for (int i=0; i<n; i++)
799 {
800 // For all entries which are not in the list
801 // write an undefined value into the LUT
802 TObject *o = names.UncheckedAt(i);
803 if (!o)
804 {
805 elt += ",0";
806 continue;
807 }
808
809 // Remove the regexp from the string which includes it
810 TString name = o->GetName();
811
812 TIter NextR(&regexp);
813 TObject *obj=0;
814 while ((obj=NextR()))
815 {
816 TPRegexp reg(obj->GetName());
817 const Ssiz_t pos = name.Index(reg, 0);
818 if (pos>0)
819 {
820 name.Remove(pos);
821 name += "-W";
822 break;
823 }
824 }
825
826 // Check if such a Key exists, if not insert it
827 const Int_t key = QueryKeyOfName(col, name);
828
829 // RESOLVE return code!!!
830 //if (key<0)
831 // return "";
832
833 // add index to the LUT
834 elt += Form(",%d", key);
835 }
836
837 // close LUT expression
838 // elt += ") AS Elt";
839 // elt += col;
840
841 elt += ") AS f";
842 elt += col;
843 elt += "KEY";
844
845 // return result
846 return elt;
847 }
848
849 TString GetELT(const char *col, TSQLResult *res, TString regexp)
850 {
851 TList list;
852 list.SetOwner();
853 list.Add(new TObjString(regexp));
854 return GetELT(col, res, list);
855 }
856
857 TString GetELTQuery(const char *col, const char *cond) const
858 {
859 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
860 "LEFT JOIN %s USING (f%sKEY) WHERE %s GROUP BY f%sName",
861 col, col, col, col, cond, col);
862 }
863
864 TString GetELTSource(const char *cond)
865 {
866 //query all sources observed in this night
867 TSQLResult *resx = Query(GetELTQuery("Source", cond));
868 if (!resx)
869 return "";
870
871 // In the case there is only a single source
872 // do not replace the source key by the ELT
873 if (resx->GetRowCount()==1)
874 return "fSourceKEY";
875
876 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
877 delete resx;
878
879 return elts;
880 }
881
882 TString GetELTProject(const char *cond)
883 {
884 //query all project names observed in this night
885 TSQLResult *resx = Query(GetELTQuery("Project", cond));
886 if (!resx)
887 return "";
888
889 // In the case there is only a single project
890 // do not replace the project key by the ELT
891 if (resx->GetRowCount()==1)
892 return "fProjectKEY";
893
894 TList regexp;
895 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
896 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
897
898 TString eltp2 = GetELT("Project", resx, regexp);
899 delete resx;
900 regexp.Delete();
901
902 return eltp2;
903 }
904*/
905 Bool_t HasAtLeastOne(TString src, TString chk) const
906 {
907 src.ToLower();
908 chk.ToLower();
909
910 for (int i=0; i<chk.Length(); i++)
911 if (src.First(chk[i])<0)
912 return kFALSE;
913
914 return kTRUE;
915 }
916
917 TString PrepareString(TSQLResult &res, TArrayL &runs)
918 {
919 // Number of result rows
920 const Int_t rows = res.GetRowCount();
921
922 runs.Set(rows); // initialize size of array for run numbers
923
924 TArrayD start(rows); // instantiate array for start times
925 TArrayD stop(rows); // instantiate array for stop times
926
927 TString str; // result string
928
929 Int_t idx=0;
930 TSQLRow *row=0;
931 while ((row=res.Next()))
932 {
933 runs[idx] = atol((*row)[0]); // run number
934
935 const TString tstart = ((*row)[2]); // start time
936 const TString tstop = ((*row)[3]); // stop time
937
938 start[idx] = MTime(tstart).GetMjd(); // convert to double
939 stop[idx] = MTime(tstop).GetMjd(); // convert to double
940
941 // This is a workaround for broken start-times
942 if (tstart=="0000-00-00 00:00:00")
943 start[idx] = stop[idx];
944
945 // Add a run-type character for this run to the string
946 str += RunType((*row)[1]);
947
948 // Increase index
949 idx++;
950 }
951/*
952 // Now the P- and D- runs are classified by the time-distance
953 // to the next or previous C-Run
954 Double_t lastc = -1;
955 Double_t nextc = -1;
956 for (int i=0; i<str.Length(); i++)
957 {
958 if (str[i]=='C')
959 {
960 // Remember stop time of C-Run as time of last C-run
961 lastc = stop[i];
962
963 // Calculate start time of next C-Run
964 const TString residual = str(i+1, str.Length());
965 const Int_t pos = residual.First('C');
966
967 // No C-Run found anymore. Finished...
968 if (pos<0)
969 break;
970
971 // Remember start time of C-Run as time of next C-run
972 nextc = start[i+1+pos];
973 continue;
974 }
975
976 // Check whether the identifying character should
977 // be converted to lower case
978 if (start[i]-lastc>nextc-stop[i])
979 str[i] = tolower(str[i]);
980 }
981*/
982 //cout << str << endl;
983 return str;
984 }
985
986 void PrintResidual(Long_t runstart, Long_t runstop, TString residual, const char *descr)
987 {
988 residual.ToLower();
989
990 // Count number of unsequences "characters"
991 const Int_t numd = residual.CountChar('d');
992 const Int_t nump = residual.CountChar('p');
993 const Int_t numb = residual.CountChar('b');
994 const Int_t numg = residual.CountChar('g');
995 const Int_t numt = residual.CountChar('t');
996 const Int_t numc = residual.CountChar('c');
997
998 // Print some information to the output steram
999 if (numd+nump+numb+numg+numt+numc==0)
1000 return;
1001
1002 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
1003 if (numd>0)
1004 cout << " (DATA!)";
1005/*
1006 if (nump==0 || numc==0 || numd==0)
1007 cout << " Missing";
1008 if (numd==0)
1009 cout << " D";
1010 if (nump==0)
1011 cout << " P";
1012 if (numc==0)
1013 cout << " C";
1014*/
1015 cout << endl;
1016 }
1017
1018 Int_t SplitBlock(Long_t runstart, Long_t runstop, const TString &cond)
1019 {
1020 // Request data necessary to split block into sequences
1021 /*const*/
1022 TString query =
1023 "SELECT fRunID, fRunTypeKEY, fRunStart, fRunStop"
1024 " FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
1025 query += Form("fRunID BETWEEN %ld AND %ld AND "
1026 /*" (fRunTypeKEY BETWEEN 2 AND 4)"*/, runstart, runstop);
1027
1028 query += cond;
1029 query += " ORDER BY fRunID";
1030
1031
1032 // Send query
1033 TSQLResult *res = Query(query);
1034 if (!res)
1035 return 2;
1036
1037 // Get String containing the sequence of P-,C- and D-Runs
1038 // and an array with the corresponding run-numbers
1039 TArrayL runs;
1040 const TString str = PrepareString(*res, runs);
1041
1042 delete res;
1043
1044 // Check if the prepared string at least contains one run of each type
1045 // ??????????????????????????????????
1046 /*
1047 if (!HasAtLeastOne(str, "PCD"))
1048 {
1049 PrintResidual(runstart, runstop, str, "Block");
1050 return kTRUE;
1051 }*/
1052 // ??????????????????????????????????
1053
1054 // Start the sequence building
1055 // ---------------------------
1056
1057 Int_t pos = 0; // Position in the string
1058 std::map<Long_t, Long_t> map; // Map for resulting sequences
1059 TString residual; // Unsequences "characters"
1060
1061 // Step through the string one character by one
1062 cout << " ";
1063 while (pos<str.Length())
1064 {
1065 Bool_t found = kFALSE;
1066
1067 // Loop over the predefined regular expressions
1068 TIter Next(&fListRegexp);
1069 Rule *obj = 0;
1070 while ((obj=(Rule*)Next()))
1071 {
1072 // Check if regular expressions matchs
1073 const Ssiz_t len = obj->Match(str, pos, runs[pos]);
1074 if (len>0)
1075 {
1076 // In case of match get the substring which
1077 // defines the sequence and print it
1078 TString sub = str(pos, len);
1079 sub.ToUpper();
1080 cout << runs[pos]<<":"<< sub << "|";
1081
1082 // Add the first and last run of the sequence to the map
1083 //map.Add(runs[pos], runs[pos+len-1]);
1084 map[runs[pos]] = runs[pos+len-1];
1085
1086 // A sequence was found...
1087 found = kTRUE;
1088
1089 // step forward in the string by the length of the sequence
1090 pos += len;
1091 break;
1092 }
1093 }
1094
1095 // If a sequence was found go on...
1096 if (found)
1097 continue;
1098
1099 // print unsequenced characters
1100 cout << (char)tolower(str[pos]);
1101
1102 // Count the number of "characters" not sequenced
1103 residual += str[pos];
1104
1105 // step one character forward
1106 pos++;
1107 }
1108 cout << endl;
1109
1110 PrintResidual(runstart, runstop, residual, "Runs ");
1111
1112 // Create all sequences which were previously found
1113 std::pair<Long_t,Long_t> drs(-1, -1);
1114 for (std::map<Long_t,Long_t>::const_iterator it=map.begin();
1115 it!=map.end(); it++)
1116 if (CreateSequence(drs, it->first, it->second, cond)==2)
1117 return 2;
1118 /*
1119 Long64_t first, last;
1120 TExMapIter iter(&map);
1121 while (iter.Next(first, last))
1122 if (CreateSequence(first, last, cond)==2)
1123 return 2;
1124 */
1125 return kTRUE;
1126 }
1127
1128 Char_t RunType(Int_t n) const
1129 {
1130 switch (n)
1131 {
1132 case 1: return 'D';
1133 case 2: return 'P'; // Ped1024
1134 case 3: return 'B'; // Ped 300
1135 case 4: return 'G'; // drs-gain
1136 case 5: return 'T'; // drs-time
1137 case 6: return 'C'; // light-pulser-ext
1138 case 11: return 'M'; // ped-and-lp-ext
1139 case 12: return 'U'; // drs-time-upshifted
1140 }
1141 return '-';
1142 }
1143 Char_t RunType(const char *str) const
1144 {
1145 return RunType(atoi(str));
1146 }
1147
1148 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks, const TString &cond)
1149 {
1150 // col key content
1151 // -----------------------------
1152 // 0 - runnumber
1153 // 1 0 RunTypeKEY
1154 // 2 1 source
1155 // 3 2 project
1156 // . .
1157 // . .
1158 // . . from transition.txt
1159 //
1160
1161 //build blocks of runs, which have the same values
1162 //for each block the first and the last run are stored in a TExMap
1163 //the values are checked with the help of an array of TStrings
1164 Long_t runstart = -1;
1165 Long_t runstop = -1;
1166
1167 const UInt_t ncols = res->GetFieldCount();
1168
1169 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
1170
1171 // Loop over runs
1172 TSQLRow *row=0;
1173 while ((row=res->Next()))
1174 {
1175 // This is the runnumber of the first run in the new block
1176 if (runstart<0)
1177 runstart=atol((*row)[0]);
1178
1179 // Check which transitions might be allowed for this run and
1180 // which are not. Check also which attributes should be ignored
1181 // for this run. The information about it is stored in the map.
1182 Int_t rc[ncols];
1183 for (UInt_t i=2; i<ncols; i++)
1184 rc[i] = CheckTransition(*res, keys, *row, i);
1185
1186 for (UInt_t i=2; i<ncols; i++)
1187 {
1188 switch (rc[i])
1189 {
1190 case kTRUE: // Transition allowed, switch to new attribute
1191 keys[i] = (*row)[i];
1192 continue;
1193
1194 case -1: // Ignore attribute, continue with next one
1195 //if (keys[i] == (*row)[i])
1196 // break; // Only ignore it if it is really different
1197 continue;
1198
1199 case kFALSE: // No decision (nothing matching found in map)
1200 break; // go on checking
1201
1202 }
1203
1204 // If past-attribute is not yet initialized, do it now.
1205 if (keys[i].IsNull())
1206 keys[i] = (*row)[i];
1207
1208 // Check whether key has changed for this run
1209 // (this condition is never true for the first run)
1210 // This condition in only reached if keys[i] was initialized
1211 if (keys[i] == (*row)[i])
1212 continue;
1213
1214 // Found one block with unique keys, fill values into TExMap
1215 // (except if this is the first run processed)
1216 cout << endl;
1217 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1218 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1219 if (SplitBlock(runstart, runstop, cond)==2)
1220 return 2;
1221 cout << " - Transition from " << RunType(keys[1]) << ":";
1222 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1223 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1224 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1225
1226 // This is already the first run of the new block
1227 runstart=atol((*row)[0]);
1228
1229 // These are the keys corresponding to the first run
1230 // in the new block. All following runs should have
1231 // identical keys. Do not set the attribute if for
1232 // this attribute the transition check evaluated
1233 // to "ignore".
1234 for (UInt_t j=2; j<ncols; j++)
1235 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1236
1237 break;
1238 }
1239
1240 keys[1] = (*row)[1]; // Remember run type for next run
1241
1242 // This is the new runnumber of the last run in this block
1243 runstop=atol((*row)[0]);
1244 }
1245
1246 // If runstart==-1 a possible block would contain a single run...
1247 if (runstart>0 && runstart!=runstop)
1248 {
1249 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1250 //fill values into TExMap (last value)
1251 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1252 if (SplitBlock(runstart, runstop, cond)==2)
1253 return 2;
1254 }
1255 cout << "Done." << endl << endl;
1256
1257 return kTRUE;
1258 }
1259
1260public:
1261 SequenceBuild(Int_t tel=1, const char *rc="sql.rc") : MSQLMagic(rc), fTelescopeNumber(tel)
1262 {
1263 fListRegexp.SetOwner();
1264
1265 // FIXME: THIS IS NOT YET HANDLED
1266 if (ReadResources("fact/resources/sequences.rc"))
1267 return;
1268 }
1269
1270 SequenceBuild(TEnv &env, Int_t tel=1) : MSQLMagic(env), fTelescopeNumber(tel)
1271 {
1272 fListRegexp.SetOwner();
1273
1274 // FIXME: THIS IS NOT YET HANDLED
1275 if (ReadResources("fact/resources/sequences.rc"))
1276 return;
1277 }
1278 ~SequenceBuild()
1279 {
1280 fMap.DeleteAll();
1281 }
1282
1283 void SetPathRawData(const char *path) { fPathRawData=path; }
1284 void SetPathSequences(const char *path) { fPathSequences=path; }
1285
1286 int Build(TString day)
1287 {
1288 cout << endl;
1289 cout << "Night of sunset at " << day << ":" << endl;
1290
1291 const TString cond = Form("fNight=%s ", day.Data());
1292
1293 // Form("fRunNumber=%s "
1294 // "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 "
1295 // "AND fTelescopeNumber=%d ",
1296 // day.Data(), fTelescopeNumber);
1297/*
1298 //query all sources observed in this night
1299 const TString elts = GetELTSource(cond);
1300 if (elts.IsNull())
1301 return 2;
1302
1303 //query all project names observed in this night
1304 const TString eltp2 = GetELTProject(cond);
1305 if (elts.IsNull())
1306 return 2;
1307*/
1308 // Setup query to get all values from the database,
1309 // that are relevant for building sequences
1310 TString query("SELECT fRunID, fRunTypeKEY ");
1311/*
1312 query += elts;
1313 query += ", ";
1314 query += eltp2;
1315*/
1316 // Now add all entries from the transition table to the query
1317 TIter NextPair(&fMap);
1318 TObject *mapkey = 0;
1319 while ((mapkey=(TPair*)NextPair()))
1320 if (!query.Contains(mapkey->GetName()))
1321 {
1322 query += ", ";
1323 query += mapkey->GetName();
1324 }
1325 query += " FROM RunInfo WHERE fExcludedFDAKEY=1 AND ";
1326 query += cond;
1327 query += " ORDER BY fRunID";
1328
1329 TSQLResult *res = Query(query);
1330 if (!res)
1331 return 2;
1332
1333 TExMap blocks;
1334 const Int_t rc = BuildBlocks(res, blocks, cond);
1335 delete res;
1336
1337 return rc;
1338 }
1339
1340 Int_t Build()
1341 {
1342 //get all dates from the database
1343 TSQLResult *res = Query("SELECT fNight FROM RunInfo GROUP BY fNight ORDER BY fNight DESC");
1344 if (!res)
1345 return 2;
1346
1347 //execute buildsequenceentries for all dates
1348 TSQLRow *row=0;
1349 while ((row=res->Next()))
1350 Build((*row)[0]);
1351
1352 delete res;
1353
1354 return 1;
1355 }
1356 ClassDef(SequenceBuild, 0)
1357};
1358
1359ClassImp(SequenceBuild);
1360
1361int buildseqentries(TString day, TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1362{
1363 SequenceBuild serv(tel, "sql.rc");
1364 if (!serv.IsConnected())
1365 {
1366 cout << "ERROR - Connection to database failed." << endl;
1367 return 0;
1368 }
1369
1370 cout << "buildseqentries" << endl;
1371 cout << "---------------" << endl;
1372 cout << endl;
1373 cout << "Connected to " << serv.GetName() << endl;
1374 if (!datapath.IsNull())
1375 cout << "DataPath: " << datapath << endl;
1376 if (!sequpath.IsNull())
1377 cout << "SeqPath: " << sequpath << endl;
1378 cout << "Day: " << day << endl;
1379 cout << "Telescope: " << tel << endl;
1380 cout << endl;
1381
1382 serv.SetIsDummy(dummy);
1383 serv.SetPathRawData(datapath);
1384 serv.SetPathSequences(sequpath);
1385 return serv.Build(day);
1386}
1387
1388//
1389// Build Sequences for all Nights
1390//
1391int buildseqentries(TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1392{
1393 SequenceBuild serv(tel, "sql.rc");
1394 if (!serv.IsConnected())
1395 {
1396 cout << "ERROR - Connection to database failed." << endl;
1397 return 0;
1398 }
1399
1400 cout << "buildseqentries" << endl;
1401 cout << "---------------" << endl;
1402 cout << endl;
1403 cout << "Connected to " << serv.GetName() << endl;
1404 cout << "DataPath: " << datapath << endl;
1405 cout << "SeqPath: " << sequpath << endl;
1406 cout << "Telescope: " << tel << endl;
1407 cout << endl;
1408
1409 serv.SetIsDummy(dummy);
1410 serv.SetPathRawData(datapath);
1411 serv.SetPathSequences(sequpath);
1412 return serv.Build();
1413}
1414
1415int buildseqentries(Int_t tel=1, Bool_t dummy=kTRUE)
1416{
1417 return buildseqentries("", "", tel, dummy);
1418}
1419
1420int buildseqentries(TString day, Int_t tel=1, Bool_t dummy=kTRUE)
1421{
1422 return buildseqentries(day, "", "", tel, dummy);
1423}
Note: See TracBrowser for help on using the repository browser.