source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 9539

Last change on this file since 9539 was 9539, checked in by snruegam, 13 years ago
*** empty log message ***
File size: 40.3 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82#include <errno.h>
83
84#include <TSQLRow.h>
85#include <TSQLResult.h>
86
87#include <TEnv.h>
88#include <TMap.h>
89#include <TMath.h>
90#include <TExMap.h>
91#include <TArrayI.h>
92#include <TArrayL.h>
93#include <TArrayD.h>
94#include <TPRegexp.h>
95#include <TSystem.h>
96#include <TObjString.h>
97#include <TObjArray.h>
98
99#include "MTime.h"
100#include "MDirIter.h"
101
102#include "MSQLMagic.h"
103
104using namespace std;
105
106 /*static*/ ULong_t GetId(const TString str)
107 {
108 const Ssiz_t dot = str.First('.');
109
110 const UInt_t run = str.Atoi();
111 const UInt_t sub = dot<0 ? 0 : atoi(str.Data()+dot+1);
112
113 return ULong_t(run)*1000+sub;
114 }
115
116
117class Rule : public TObject
118{
119private:
120 TPRegexp fRegexp;
121
122 ULong_t fMin;
123 ULong_t fMax;
124
125public:
126 Rule(TObjArray &arr) :
127 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
128 fMin(0), fMax((ULong_t)-1)
129 {
130 if (arr.GetEntries()>1)
131 fMin = GetId(arr[1]->GetName());
132 if (arr.GetEntries()>2)
133 fMax = GetId(arr[2]->GetName());
134 }
135
136 Ssiz_t Match(const TString &str, Int_t idx, Long_t run=-1)
137 {
138 if (!IsValid(run))
139 return 0;
140
141 TString mods;
142 TArrayI pos;
143 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
144
145 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
146 }
147
148 Bool_t IsValid(ULong_t run) const { return run>=fMin && run<=fMax; }
149 ClassDef(Rule, 0)
150};
151ClassImp(Rule);
152
153class CheckMatch : public TObject
154{
155private:
156 TPRegexp fRunType1;
157 TPRegexp fRunType2;
158
159 TPRegexp fRegexp1;
160 TPRegexp fRegexp2;
161
162 ULong_t fMin;
163 ULong_t fMax;
164
165 void Init(const TObjArray &arr, Int_t idx=0)
166 {
167 const Int_t n = arr.GetEntries();
168
169 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
170
171 for (int i=0; i<n-idx; i++)
172 {
173 //cout << arr[i+idx]->GetName() << " ";
174
175 TString str(arr[i+idx]->GetName());
176 if (str=="*")
177 str = ".*";
178
179 switch (isminus && i>1 ? i+1 : i)
180 {
181 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
182 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
183 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
184 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
185 case 4: fMin = GetId(str); break;
186 case 5: fMax = GetId(str); break;
187 }
188 }
189 }
190
191public:
192 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
193
194 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
195 {
196 TObjArray *arr = txt.Tokenize(" ");
197 Init(*arr);
198 delete arr;
199 }
200
201 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((ULong_t)-1)
202 {
203 Init(arr, idx);
204 }
205
206 CheckMatch(const char *from, const char *to, Long_t min=0, Long_t max=-1)
207 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
208 {
209 }
210
211 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, ULong_t run=0)
212 {
213 if (run>0)
214 {
215 if (run<fMin || run>fMax)
216 return kFALSE;
217 }
218
219 const TString test("X-"); // FIXME:STUPID!
220
221 //if (test.Index(fRunType2)==1)
222 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
223
224 if (test.Index(fRunType1,0)==1)
225 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
226
227 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
228 }
229 ClassDef(CheckMatch,0)
230};
231ClassImp(CheckMatch);
232
233class CheckList : public TList
234{
235public:
236 CheckList() { SetOwner(); }
237 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Long_t run=-1) const
238 {
239 TIter Next(this);
240
241 CheckMatch *check = 0;
242
243 while ((check=(CheckMatch*)Next()))
244 {
245 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
246 if (rc)
247 return rc;
248 }
249
250 return kFALSE;
251 }
252 ClassDef(CheckList,0)
253};
254ClassImp(CheckList);
255
256class SequenceBuild : public MSQLMagic
257{
258private:
259 TString fPathRawData;
260 TString fPathSequences;
261
262 Int_t fTelescopeNumber;
263
264 TMap fMap;
265 TList fListRegexp;
266
267 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
268 {
269 // Get the name of the column from result table
270 const TString key = res.GetFieldName(i);
271
272 // Get the list with the allowed attributed from the map
273 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
274 if (!list)
275 return kFALSE;
276
277 // Check whether the current run (row[0]) with the current attribute
278 // (row[i]) and the current run-type (row[1]) matches the attribute
279 // of the last run (keys[i] with run-type row[i])
280 return list->Matches(keys[1], row[1], keys[i], row[i], atol(row[0]));
281 }
282
283 Bool_t InsertSequence(Long_t from, Long_t to)
284 {
285 cout << " - Inserting Sequence into database." << endl;
286
287 // ========== Request number of events ==========
288
289 // Can be replaced by
290
291 const TString runtime =
292 "SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))), ";
293
294 const TString where = Form("(fRunNumber*1000+fFileNumber BETWEEN %ld AND %ld) "
295 "AND fTelescopeNumber=%d AND fExcludedFDAKEY=1",
296 from, to, fTelescopeNumber);
297
298 TString query;
299 query = "SELECT SUM(fNumEvents), ";
300 query += runtime;
301 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
302 query += " MIN(fAzimuth), MAX(fAzimuth), ";
303 query += " MIN(if(fRunStart=0,NULL,fRunStart)), ";
304 query += " MAX(if(fRunStop=0,NULL,fRunStop)), ";
305 query += " ELT(MAX(FIELD(fLightConditionsKEY, 1, 2, 7, 5, 8)), 1, 2, 7, 5, 8) ";
306 query += " FROM RunData WHERE ";
307 query += where;
308 query += " AND fRunTypeKEY=2";
309
310 TSQLResult *res = Query(query);
311 if (!res)
312 return kFALSE;
313
314 TSQLRow *row = res->Next();
315 if (!row || res->GetFieldCount()!=9)
316 {
317 cout << "ERROR - Wrong result from query: " << query << endl;
318 return kFALSE;
319 }
320
321 const TString nevts = (*row)[0];
322 const TString secs = (*row)[1];
323 const TString zdmin = (*row)[2];
324 const TString zdmax = (*row)[3];
325 const TString azmin = (*row)[4];
326 const TString azmax = (*row)[5];
327 const TString start = (*row)[6];
328 const TString stop = (*row)[7];
329 const TString light = (*row)[8];
330
331 delete res;
332
333 const TString elts = GetELTSource(where);
334 if (elts.IsNull())
335 return kFALSE;
336
337 const TString eltp = GetELTProject(where);
338 if (eltp.IsNull())
339 return kFALSE;
340
341 // ========== Request data of sequence ==========
342 query = "SELECT ";
343 query += elts;
344 query += ", ";
345 query += eltp;
346 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
347 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
348 " fTriggerDelayTableKEY, fObservationModeKEY, fSumTriggerFlagKEY "
349 " FROM RunData WHERE fRunTypeKEY=2 AND ";
350 query += where;
351 query += " LIMIT 1";
352
353 res = Query(query);
354 if (!res)
355 return kFALSE;
356
357 row = res->Next();
358 if (!row || res->GetFieldCount()!=9)
359 {
360 cout << "ERROR - No result from query: " << query << endl;
361 return kFALSE;
362 }
363
364 const TString set = Form("fSequenceFirst=%ld, fTelescopeNumber=%d ", from/1000, fTelescopeNumber);
365
366 TString query1;
367 query1 += set;
368 query1 += Form(",fSequenceLast=%ld,", to/1000);
369 query1 += Form(" fSourceKEY=%s,", (*row)[0]);
370 query1 += Form(" fProjectKEY=%s,", (*row)[1]);
371 query1 += Form(" fNumEvents=%s,", nevts.Data());
372 query1 += Form(" fRunTime=%s,", secs.Data());
373 query1 += Form(" fRunStart=\"%s\",", start.Data());
374 query1 += Form(" fRunStop=\"%s\",", stop.Data());
375 query1 += Form(" fZenithDistanceMin=%s,", zdmin.Data());
376 query1 += Form(" fZenithDistanceMax=%s,", zdmax.Data());
377 query1 += Form(" fAzimuthMin=%s,", azmin.Data());
378 query1 += Form(" fAzimuthMax=%s,", azmax.Data());
379 query1 += Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
380 query1 += Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
381 query1 += Form(" fHvSettingsKEY=%s,", (*row)[4]);
382 query1 += Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
383 query1 += Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
384 query1 += Form(" fLightConditionsKEY=%s,", light.Data());
385 query1 += Form(" fObservationModeKEY=%s, ", (*row)[7]);
386 query1 += Form(" fSumTriggerFlagKEY=%s, ", (*row)[8]);
387 query1 += "fManuallyChangedKEY=1";
388
389 delete res;
390
391 const TString where2 = Form("(fRunTypeKEY BETWEEN 2 AND 4) AND %s",
392 where.Data());
393
394 if (!Insert("Sequences", query1))
395 {
396 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
397 return kFALSE;
398 }
399
400 if (!Update("RunData", set, where))
401 {
402 cout << "ERROR - Could not update RunData." << endl;
403 return kFALSE;
404 }
405
406 TString prio = set;
407 prio += Form(", fPriority=%ld ", from/1000);
408 if (!Insert("SequenceProcessStatus", prio))
409 {
410 cout << "ERROR - Could not insert Sequence into SequenceProcessStatus." << endl;
411 return kFALSE;
412 }
413
414 return kTRUE;
415 }
416
417 Bool_t DeleteSequence(Int_t sequ)
418 {
419 if (fPathRawData.IsNull() || fPathSequences.IsNull())
420 {
421 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
422 return kTRUE;
423 }
424
425 //queries to delete information from the database
426 const TString query(Form("fSequenceFirst=%d", sequ));
427
428 //commands to delete files from the disk
429 const TString fname(Form("%s/%04d/sequence%08d.txt", fPathSequences.Data(),sequ/10000, sequ));
430 const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
431 const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
432
433 if (!Delete("Calibration", query))
434 return 2;
435
436 if (!Delete("Star", query))
437 return 2;
438
439 if (!Delete("SequenceProcessStatus", query))
440 return 2;
441
442 if (!Delete("Sequences", query))
443 return 2;
444
445 if (!Update("RunData", "fSequenceFirst=0", query))
446 return 2;
447
448 if (IsDummy())
449 {
450 cout << " + would unlink " << fname << endl;
451 cout << " + would remove " << cmd1 << endl;
452 cout << " + would remove " << cmd2 << endl;
453 return kTRUE;
454 }
455
456 cout << " + will unlink " << fname << endl;
457 cout << " + will remove " << cmd1 << endl;
458 cout << " + will remove " << cmd2 << endl;
459
460 gSystem->Unlink(fname);
461
462 gSystem->Exec(cmd1);
463 gSystem->Exec(cmd2);
464
465 return kTRUE;
466 }
467
468 Int_t CheckSequence(Long_t runstart, Long_t runstop)
469 {
470 const char *fmt1 = "SELECT fRunNumber*1000+fFileNumber AS Id FROM RunData WHERE";
471 const char *fmt2 = "AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) ORDER BY Id";
472
473 const TString query1 = Form("%s fTelescopeNumber=%d AND fSequenceFirst=%ld %s", fmt1, fTelescopeNumber, runstart/1000, fmt2);
474 const TString query2 = Form("%s fTelescopeNumber=%d AND fRunNumber*1000+fFileNumber BETWEEN %ld AND %ld %s", fmt1, fTelescopeNumber, runstart, runstop, fmt2);
475
476 TSQLResult *res1 = Query(query1);
477 if (!res1)
478 return 2;
479
480 TSQLResult *res2 = Query(query2);
481 if (!res2)
482 {
483 delete res1;
484 return 2;
485 }
486
487 while (1)
488 {
489 TSQLRow *row1 = res1->Next();
490 TSQLRow *row2 = res2->Next();
491
492 if (!row1 && !row2)
493 return kTRUE;
494
495 if (!row1 || !row2)
496 return kFALSE;
497
498 if (atol((*row1)[0])!=atol((*row2)[0]))
499 return kFALSE;
500 }
501
502 return kFALSE;
503 }
504
505 Int_t CreateSequence(Long_t runstart, Long_t runstop)
506 {
507 cout << " * Creating Sequence " << runstart << "-" << runstop << ":" << endl;
508
509 TString query=
510 Form("SELECT fSequenceFirst FROM RunData "
511 " WHERE fRunNumber*1000+fFileNumber BETWEEN %ld AND %ld AND "
512 " fTelescopeNumber=%d AND fSequenceFirst>0 AND "
513 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
514 " GROUP BY fSequenceFirst", runstart, runstop,
515 fTelescopeNumber);
516
517 TSQLResult *res = Query(query);
518 if (!res)
519 return 2;
520
521 const Int_t cnt = res->GetRowCount();
522
523 Int_t rc = kTRUE;
524 if (cnt==1)
525 {
526 TSQLRow *row=res->Next();
527 const Int_t check = CheckSequence(runstart, runstop);
528 if (check==kTRUE)
529 {
530 cout << " - Identical sequence already existing." << endl;
531 delete res;
532 return kTRUE;
533 }
534 if (check==2)
535 rc=2;
536 else
537 {
538 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
539 if (DeleteSequence(atoi((*row)[0]))==2)
540 rc = 2;
541 }
542 }
543 else
544 {
545 TSQLRow *row=0;
546 while ((row=res->Next()))
547 {
548 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
549 if (DeleteSequence(atoi((*row)[0]))==2)
550 rc = 2;
551 }
552 }
553
554 delete res;
555
556 if (rc==2)
557 return 2;
558
559 if (!InsertSequence(runstart, runstop))
560 return 2;
561
562 return kTRUE;
563 }
564
565 Bool_t ReadResources(const char *fname)
566 {
567 // Check for the section header
568 TPRegexp regexp("^\\[(Transition|Regexp):?[0-9 ]*\\]$");
569 // Check if section header contains a number
570 TPRegexp regnum("[0-9]");
571 // Check if section header contains the telescope number
572 TPRegexp regtel(Form("[^0-9]0*%d[^0-9]", fTelescopeNumber));
573
574 ifstream fin(fname);
575 if (!fin)
576 {
577 cout << "Cannot open file " << fname << ": ";
578 cout << strerror(errno) << endl;
579 return kFALSE;
580 }
581
582 Int_t section = 0;
583
584 TString key;
585 while (1)
586 {
587 TString txt;
588 txt.ReadLine(fin);
589 if (!fin)
590 break;
591
592 txt = txt.Strip(TString::kBoth);
593
594 if (txt[0]=='#' || txt.IsNull())
595 continue;
596
597 if (txt[0]=='[')
598 {
599 TString sec = txt(regexp);
600 if (!sec.IsNull())
601 {
602 // Skip sections with the wrong telescope number
603 // (If section contains numbers but not the tel num)
604 if (!sec(regnum).IsNull() && sec(regtel).IsNull())
605 continue;
606
607 // Check which section we are in
608 if (sec.BeginsWith("[Transition"))
609 section = 1;
610 if (sec.BeginsWith("[Regexp"))
611 section = 2;
612
613 continue;
614 }
615
616 if (section!=2)
617 {
618 cout << "WARNING - Line starts with [ but we are not in the Regexp section." << endl;
619 cout << txt << endl;
620 continue;
621 }
622 }
623
624
625 TObjArray *arr = txt.Tokenize(" ");
626
627 if (arr->GetEntries()>0)
628 switch (section)
629 {
630 case 1:
631 {
632 TString key = arr->At(0)->GetName();
633 key.Prepend("f");
634 key.Append("KEY");
635
636 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
637 if (!list)
638 {
639 //cout << key << endl;
640 list = new CheckList;
641 fMap.Add(new TObjString(key), list);
642 }
643
644 if (arr->GetEntries()>1)
645 {
646 //cout << key << " ";
647 list->Add(new CheckMatch(*arr, 1));
648 }
649 }
650 break;
651 case 2:
652 fListRegexp.Add(new Rule(*arr));
653 break;
654 }
655
656 delete arr;
657 }
658
659 return kTRUE;
660 }
661
662 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
663 {
664 TObjArray names; // array with old names (including regexp)
665
666 // Add to array and expand the array if necessary
667 TSQLRow *row=0;
668 while ((row=res->Next()))
669 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
670
671 // Now a LUT is build which converts the keys for
672 // the names including the regexp into keys for
673 // the names excluding the regexp
674 TString elt(Form("ELT(RunData.f%sKEY+1", col));
675
676 // loop over all entries in the list
677 const Int_t n = names.GetSize();
678 for (int i=0; i<n; i++)
679 {
680 // For all entries which are not in the list
681 // write an undefined value into the LUT
682 TObject *o = names.UncheckedAt(i);
683 if (!o)
684 {
685 elt += ",0";
686 continue;
687 }
688
689 // Remove the regexp from the string which includes it
690 TString name = o->GetName();
691
692 TIter NextR(&regexp);
693 TObject *obj=0;
694 while ((obj=NextR()))
695 {
696 TPRegexp reg(obj->GetName());
697 const Ssiz_t pos = name.Index(reg, 0);
698 if (pos>0)
699 {
700 name.Remove(pos);
701 name += "-W";
702 break;
703 }
704 }
705
706 // Check if such a Key exists, if not insert it
707 const Int_t key = QueryKeyOfName(col, name);
708
709 // RESOLVE return code!!!
710 //if (key<0)
711 // return "";
712
713 // add index to the LUT
714 elt += Form(",%d", key);
715 }
716
717 // close LUT expression
718 // elt += ") AS Elt";
719 // elt += col;
720
721 elt += ") AS f";
722 elt += col;
723 elt += "KEY";
724
725 // return result
726 return elt;
727 }
728
729 TString GetELT(const char *col, TSQLResult *res, TString regexp)
730 {
731 TList list;
732 list.SetOwner();
733 list.Add(new TObjString(regexp));
734 return GetELT(col, res, list);
735 }
736
737 TString GetELTQuery(const char *col, const char *cond) const
738 {
739 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
740 "LEFT JOIN %s USING (f%sKEY) WHERE %s GROUP BY f%sName",
741 col, col, col, col, cond, col);
742 }
743
744 TString GetELTSource(const char *cond)
745 {
746 //query all sources observed in this night
747 TSQLResult *resx = Query(GetELTQuery("Source", cond));
748 if (!resx)
749 return "";
750
751 // In the case there is only a single source
752 // do not replace the source key by the ELT
753 if (resx->GetRowCount()==1)
754 return "fSourceKEY";
755
756 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
757 delete resx;
758
759 return elts;
760 }
761
762 TString GetELTProject(const char *cond)
763 {
764 //query all project names observed in this night
765 TSQLResult *resx = Query(GetELTQuery("Project", cond));
766 if (!resx)
767 return "";
768
769 // In the case there is only a single project
770 // do not replace the project key by the ELT
771 if (resx->GetRowCount()==1)
772 return "fProjectKEY";
773
774 TList regexp;
775 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
776 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
777
778 TString eltp2 = GetELT("Project", resx, regexp);
779 delete resx;
780 regexp.Delete();
781
782 return eltp2;
783 }
784
785 Bool_t HasAtLeastOne(TString src, TString chk) const
786 {
787 src.ToLower();
788 chk.ToLower();
789
790 for (int i=0; i<chk.Length(); i++)
791 if (src.First(chk[i])<0)
792 return kFALSE;
793
794 return kTRUE;
795 }
796
797 TString PrepareString(TSQLResult &res, TArrayL &runs)
798 {
799 // Number of result rows
800 const Int_t rows = res.GetRowCount();
801
802 runs.Set(rows); // initialize size of array for run numbers
803
804 TArrayD start(rows); // instantiate array for start times
805 TArrayD stop(rows); // instantiate array for stop times
806
807 TString str; // result string
808
809 Int_t idx=0;
810 TSQLRow *row=0;
811 while ((row=res.Next()))
812 {
813 runs[idx] = atol((*row)[0]); // run number
814
815 const TString tstart = ((*row)[2]); // start time
816 const TString tstop = ((*row)[3]); // stop time
817
818 start[idx] = MTime(tstart).GetMjd(); // convert to double
819 stop[idx] = MTime(tstop).GetMjd(); // convert to double
820
821 // This is a workaround for broken start-times
822 if (tstart=="0000-00-00 00:00:00")
823 start[idx] = stop[idx];
824
825 // Add a run-type character for this run to the string
826 str += RunType((*row)[1]);
827
828 // Increase index
829 idx++;
830 }
831
832 // Now the P- and D- runs are classified by the time-distance
833 // to the next or previous C-Run
834 Double_t lastc = -1;
835 Double_t nextc = -1;
836 for (int i=0; i<str.Length(); i++)
837 {
838 if (str[i]=='C')
839 {
840 // Remember stop time of C-Run as time of last C-run
841 lastc = stop[i];
842
843 // Calculate start time of next C-Run
844 const TString residual = str(i+1, str.Length());
845 const Int_t pos = residual.First('C');
846
847 // No C-Run found anymore. Finished...
848 if (pos<0)
849 break;
850
851 // Remember start time of C-Run as time of next C-run
852 nextc = start[i+1+pos];
853 continue;
854 }
855
856 // Check whether the identifying character should
857 // be converted to lower case
858 if (start[i]-lastc>nextc-stop[i])
859 str[i] = tolower(str[i]);
860 }
861 //cout << str << endl;
862 return str;
863 }
864
865 void PrintResidual(Long_t runstart, Long_t runstop, TString residual, const char *descr)
866 {
867 residual.ToLower();
868
869 // Count number of unsequences "characters"
870 const Int_t nump = residual.CountChar('p');
871 const Int_t numd = residual.CountChar('d');
872 const Int_t numc = residual.CountChar('c');
873
874 // Print some information to the output steram
875 if (nump+numc+numd==0)
876 return;
877
878 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
879 if (numd>0)
880 cout << " (DATA!)";
881
882 if (nump==0 || numc==0 || numd==0)
883 cout << " Missing";
884 if (numd==0)
885 cout << " D";
886 if (nump==0)
887 cout << " P";
888 if (numc==0)
889 cout << " C";
890
891 cout << endl;
892 }
893
894 Int_t SplitBlock(Long_t runstart, Long_t runstop, const TString &cond)
895 {
896 // Request data necessary to split block into sequences
897 /*const*/ TString query=
898 Form("SELECT fRunNumber*1000+fFileNumber AS Id, fRunTypeKEY, fRunStart, fRunStop"
899 " FROM RunData "
900 " WHERE fRunNumber*1000+fFileNumber BETWEEN %ld AND %ld AND "
901 /*" fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
902 " ORDER BY Id"*/, runstart, runstop);
903
904 query += cond;
905 query += " ORDER BY Id";
906
907
908 // Send query
909 TSQLResult *res = Query(query);
910 if (!res)
911 return 2;
912
913 // Get String containing the sequence of P-,C- and D-Runs
914 // and an array with the corresponding run-numbers
915 TArrayL runs;
916 const TString str = PrepareString(*res, runs);
917
918 delete res;
919
920 // Check if the prepared string at least contains one run of each type
921 if (!HasAtLeastOne(str, "PCD"))
922 {
923 PrintResidual(runstart, runstop, str, "Block");
924 return kTRUE;
925 }
926
927 // Start the sequence building
928 // ---------------------------
929
930 Int_t pos = 0; // Position in the string
931 TExMap map; // Map for resulting sequences
932 TString residual; // Unsequences "characters"
933
934 // Step through the string one character by one
935 cout << " ";
936 while (pos<str.Length())
937 {
938 Bool_t found = kFALSE;
939
940 // Loop over the predefined regular expressions
941 TIter Next(&fListRegexp);
942 Rule *obj = 0;
943 while ((obj=(Rule*)Next()))
944 {
945 // Check if regular expressions matchs
946 const Ssiz_t len = obj->Match(str, pos, runs[pos]);
947 if (len>0)
948 {
949 // In case of match get the substring which
950 // defines the sequence and print it
951 TString sub = str(pos, len);
952 sub.ToUpper();
953 cout << runs[pos]<<":"<< sub << "|";
954
955 // Add the first and last run of the sequence to the map
956 map.Add(runs[pos], runs[pos+len-1]);
957
958 // A sequence was found...
959 found = kTRUE;
960
961 // step forward in the string by the length of the sequence
962 pos += len;
963 break;
964 }
965 }
966
967 // If a sequence was found go on...
968 if (found)
969 continue;
970
971 // print unsequenced characters
972 cout << (char)tolower(str[pos]);
973
974 // Count the number of "characters" not sequenced
975 residual += str[pos];
976
977 // step one character forward
978 pos++;
979 }
980 cout << endl;
981
982 PrintResidual(runstart, runstop, residual, "Runs ");
983
984 // Create all sequences which were previously found
985 Long_t first, last;
986 TExMapIter iter(&map);
987 while (iter.Next(first, last))
988 if (CreateSequence(first, last)==2)
989 return 2;
990
991 return kTRUE;
992 }
993
994 Char_t RunType(Int_t n) const
995 {
996 switch (n)
997 {
998 case 2: return 'D';
999 case 3: return 'P';
1000 case 4: return 'C';
1001 }
1002 return '-';
1003 }
1004 Char_t RunType(const char *str) const
1005 {
1006 return RunType(atoi(str));
1007 }
1008
1009 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks, const TString &cond)
1010 {
1011 // col key content
1012 // -----------------------------
1013 // 0 - runnumber
1014 // 1 0 RunTypeKEY
1015 // 2 1 source
1016 // 3 2 project
1017 // . .
1018 // . .
1019 // . . from transition.txt
1020 //
1021
1022 //build blocks of runs, which have the same values
1023 //for each block the first and the last run are stored in a TExMap
1024 //the values are checked with the help of an array of TStrings
1025 Long_t runstart = -1;
1026 Long_t runstop = -1;
1027
1028 const UInt_t ncols = res->GetFieldCount();
1029
1030 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
1031
1032 // Loop over runs
1033 TSQLRow *row=0;
1034 while ((row=res->Next()))
1035 {
1036 // This is the runnumber of the first run in the new block
1037 if (runstart<0)
1038 runstart=atol((*row)[0]);
1039
1040 // Check which transitions might be allowed for this run and
1041 // which are not. Check also which attributes should be ignored
1042 // for this run. The information about it is stored in the map.
1043 Int_t rc[ncols];
1044 for (UInt_t i=2; i<ncols; i++)
1045 rc[i] = CheckTransition(*res, keys, *row, i);
1046
1047 for (UInt_t i=2; i<ncols; i++)
1048 {
1049 switch (rc[i])
1050 {
1051 case kTRUE: // Transition allowed, switch to new attribute
1052 keys[i] = (*row)[i];
1053 continue;
1054
1055 case -1: // Ignore attribute, continue with next one
1056 //if (keys[i] == (*row)[i])
1057 // break; // Only ignore it if it is really different
1058 continue;
1059
1060 case kFALSE: // No decision (nothing matching found in map)
1061 break; // go on checking
1062
1063 }
1064
1065 // If past-attribute is not yet initialized, do it now.
1066 if (keys[i].IsNull())
1067 keys[i] = (*row)[i];
1068
1069 // Check whether key has changed for this run
1070 // (this condition is never true for the first run)
1071 // This condition in only reached if keys[i] was initialized
1072 if (keys[i] == (*row)[i])
1073 continue;
1074
1075 // Found one block with unique keys, fill values into TExMap
1076 // (except if this is the first run processed)
1077 cout << endl;
1078 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1079 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1080 if (SplitBlock(runstart, runstop, cond)==2)
1081 return 2;
1082 cout << " - Transition from " << RunType(keys[1]) << ":";
1083 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1084 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1085 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1086
1087 // This is already the first run of the new block
1088 runstart=atol((*row)[0]);
1089
1090 // These are the keys corresponding to the first run
1091 // in the new block. All following runs should have
1092 // identical keys. Do not set the attribute if for
1093 // this attribute the transition check evaluated
1094 // to "ignore".
1095 for (UInt_t j=2; j<ncols; j++)
1096 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1097
1098 break;
1099 }
1100
1101 keys[1] = (*row)[1]; // Remember run type for next run
1102
1103 // This is the new runnumber of the last run in this block
1104 runstop=atol((*row)[0]);
1105 }
1106
1107 // If runstart==-1 a possible block would contain a single run...
1108 if (runstart>0 && runstart!=runstop)
1109 {
1110 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1111 //fill values into TExMap (last value)
1112 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1113 if (SplitBlock(runstart, runstop, cond)==2)
1114 return 2;
1115 }
1116 cout << "Done." << endl << endl;
1117
1118 return kTRUE;
1119 }
1120
1121public:
1122 SequenceBuild(Int_t tel=1, const char *rc="sql.rc") : MSQLMagic(rc), fTelescopeNumber(tel)
1123 {
1124 fListRegexp.SetOwner();
1125
1126 // FIXME: THIS IS NOT YET HANDLED
1127 if (ReadResources("resources/sequences.rc"))
1128 return;
1129 }
1130
1131 SequenceBuild(TEnv &env, Int_t tel=1) : MSQLMagic(env), fTelescopeNumber(tel)
1132 {
1133 fListRegexp.SetOwner();
1134
1135 // FIXME: THIS IS NOT YET HANDLED
1136 if (ReadResources("resources/sequences.rc"))
1137 return;
1138 }
1139 ~SequenceBuild()
1140 {
1141 fMap.DeleteAll();
1142 }
1143
1144 void SetPathRawData(const char *path) { fPathRawData=path; }
1145 void SetPathSequences(const char *path) { fPathSequences=path; }
1146
1147 int Build(TString day)
1148 {
1149 cout << endl;
1150 cout << "Night of sunrise at " << day << ":" << endl;
1151
1152 day += " 13:00:00";
1153 const TString cond =
1154 Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\") "
1155 "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 "
1156 "AND fTelescopeNumber=%d ",
1157 day.Data(), day.Data(), fTelescopeNumber);
1158
1159 //query all sources observed in this night
1160 const TString elts = GetELTSource(cond);
1161 if (elts.IsNull())
1162 return 2;
1163
1164 //query all project names observed in this night
1165 const TString eltp2 = GetELTProject(cond);
1166 if (elts.IsNull())
1167 return 2;
1168
1169 // Setup query to get all values from the database,
1170 // that are relevant for building sequences
1171 TString query("SELECT fRunNumber*1000+fFileNumber AS Id, fRunTypeKEY, ");
1172 query += elts;
1173 query += ", ";
1174 query += eltp2;
1175
1176 // Now add all entries from the transition table to the query
1177 TIter NextPair(&fMap);
1178 TObject *mapkey = 0;
1179 while ((mapkey=(TPair*)NextPair()))
1180 if (!query.Contains(mapkey->GetName()))
1181 {
1182 query += ", ";
1183 query += mapkey->GetName();
1184 }
1185 query += Form(" FROM RunData WHERE %s ORDER BY Id", cond.Data());
1186
1187 TSQLResult *res = Query(query);
1188 if (!res)
1189 return 2;
1190
1191 TExMap blocks;
1192 const Int_t rc = BuildBlocks(res, blocks, cond);
1193 delete res;
1194
1195 return rc;
1196 }
1197
1198 Int_t Build()
1199 {
1200 //get all dates from the database
1201 TSQLResult *res = Query("SELECT fDate FROM SequenceBuildStatus ORDER BY fDate DESC");
1202 if (!res)
1203 return 2;
1204
1205 //execute buildsequenceentries for all dates
1206 TSQLRow *row=0;
1207 while ((row=res->Next()))
1208 Build((*row)[0]);
1209
1210 delete res;
1211
1212 return 1;
1213 }
1214 ClassDef(SequenceBuild, 0)
1215};
1216
1217ClassImp(SequenceBuild);
1218
1219int buildsequenceentries(TString day, TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1220{
1221 SequenceBuild serv(tel, "sql.rc");
1222 if (!serv.IsConnected())
1223 {
1224 cout << "ERROR - Connection to database failed." << endl;
1225 return 0;
1226 }
1227
1228 cout << "buildsequenceentries" << endl;
1229 cout << "--------------------" << endl;
1230 cout << endl;
1231 cout << "Connected to " << serv.GetName() << endl;
1232 if (!datapath.IsNull())
1233 cout << "DataPath: " << datapath << endl;
1234 if (!sequpath.IsNull())
1235 cout << "SeqPath: " << sequpath << endl;
1236 cout << "Day: " << day << endl;
1237 cout << "Telescope: " << tel << endl;
1238 cout << endl;
1239
1240 serv.SetIsDummy(dummy);
1241 serv.SetPathRawData(datapath);
1242 serv.SetPathSequences(sequpath);
1243 return serv.Build(day);
1244}
1245
1246//
1247// Build Sequences for all Nights
1248//
1249int buildsequenceentries(TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1250{
1251 SequenceBuild serv(tel, "sql.rc");
1252 if (!serv.IsConnected())
1253 {
1254 cout << "ERROR - Connection to database failed." << endl;
1255 return 0;
1256 }
1257
1258 cout << "buildsequenceentries" << endl;
1259 cout << "--------------------" << endl;
1260 cout << endl;
1261 cout << "Connected to " << serv.GetName() << endl;
1262 cout << "DataPath: " << datapath << endl;
1263 cout << "SeqPath: " << sequpath << endl;
1264 cout << "Telescope: " << tel << endl;
1265 cout << endl;
1266
1267 serv.SetIsDummy(dummy);
1268 serv.SetPathRawData(datapath);
1269 serv.SetPathSequences(sequpath);
1270 return serv.Build();
1271}
1272
1273int buildsequenceentries(Int_t tel=1, Bool_t dummy=kTRUE)
1274{
1275 return buildsequenceentries("", "", tel, dummy);
1276}
1277
1278int buildsequenceentries(TString day, Int_t tel=1, Bool_t dummy=kTRUE)
1279{
1280 return buildsequenceentries(day, "", "", tel, dummy);
1281}
Note: See TracBrowser for help on using the repository browser.