source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 9213

Last change on this file since 9213 was 9213, checked in by hoehne, 13 years ago
*** empty log message ***
File size: 40.2 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2008
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82#include <errno.h>
83
84#include <TSQLRow.h>
85#include <TSQLResult.h>
86
87#include <TEnv.h>
88#include <TMap.h>
89#include <TMath.h>
90#include <TExMap.h>
91#include <TArrayI.h>
92#include <TArrayD.h>
93#include <TPRegexp.h>
94#include <TSystem.h>
95#include <TObjString.h>
96#include <TObjArray.h>
97
98#include "MTime.h"
99#include "MDirIter.h"
100
101#include "MSQLMagic.h"
102
103using namespace std;
104
105 /*static*/ UInt_t GetId(const TString str)
106 {
107 const Ssiz_t dot = str.First('.');
108
109 const UInt_t run = str.Atoi();
110 const UInt_t sub = dot<0 ? 0 : atoi(str.Data()+dot+1);
111
112 return run*1000+sub;
113 }
114
115
116class Rule : public TObject
117{
118private:
119 TPRegexp fRegexp;
120
121 UInt_t fMin;
122 UInt_t fMax;
123
124public:
125 Rule(TObjArray &arr) :
126 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
127 fMin(0), fMax((UInt_t)-1)
128 {
129 if (arr.GetEntries()>1)
130 fMin = GetId(arr[1]->GetName());
131 if (arr.GetEntries()>2)
132 fMax = GetId(arr[2]->GetName());
133 }
134
135 Ssiz_t Match(const TString &str, Int_t idx, Int_t run=-1)
136 {
137 if (!IsValid(run))
138 return 0;
139
140 TString mods;
141 TArrayI pos;
142 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
143
144 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
145 }
146
147 Bool_t IsValid(UInt_t run) const { return run<0 || (run>=fMin && run<=fMax); }
148 ClassDef(Rule, 0)
149};
150ClassImp(Rule);
151
152class CheckMatch : public TObject
153{
154private:
155 TPRegexp fRunType1;
156 TPRegexp fRunType2;
157
158 TPRegexp fRegexp1;
159 TPRegexp fRegexp2;
160
161 UInt_t fMin;
162 UInt_t fMax;
163
164 void Init(const TObjArray &arr, Int_t idx=0)
165 {
166 const Int_t n = arr.GetEntries();
167
168 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
169
170 for (int i=0; i<n-idx; i++)
171 {
172 //cout << arr[i+idx]->GetName() << " ";
173
174 TString str(arr[i+idx]->GetName());
175 if (str=="*")
176 str = ".*";
177
178 switch (isminus && i>1 ? i+1 : i)
179 {
180 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
181 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
182 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
183 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
184 case 4: fMin = GetId(str); break;
185 case 5: fMax = GetId(str); break;
186 }
187 }
188 }
189
190public:
191 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
192
193 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
194 {
195 TObjArray *arr = txt.Tokenize(" ");
196 Init(*arr);
197 delete arr;
198 }
199
200 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
201 {
202 Init(arr, idx);
203 }
204
205 CheckMatch(const char *from, const char *to, Int_t min=0, Int_t max=-1)
206 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
207 {
208 }
209
210 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, UInt_t run=0)
211 {
212 if (run>0)
213 {
214 if (run<fMin || run>fMax)
215 return kFALSE;
216 }
217
218 const TString test("X-"); // FIXME:STUPID!
219
220 //if (test.Index(fRunType2)==1)
221 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
222
223 if (test.Index(fRunType1,0)==1)
224 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
225
226 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
227 }
228 ClassDef(CheckMatch,0)
229};
230ClassImp(CheckMatch);
231
232class CheckList : public TList
233{
234public:
235 CheckList() { SetOwner(); }
236 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Int_t run=-1) const
237 {
238 TIter Next(this);
239
240 CheckMatch *check = 0;
241
242 while ((check=(CheckMatch*)Next()))
243 {
244 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
245 if (rc)
246 return rc;
247 }
248
249 return kFALSE;
250 }
251 ClassDef(CheckList,0)
252};
253ClassImp(CheckList);
254
255class SequenceBuild : public MSQLMagic
256{
257private:
258 TString fPathRawData;
259 TString fPathSequences;
260
261 Int_t fTelescopeNumber;
262
263 TMap fMap;
264 TList fListRegexp;
265
266 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
267 {
268 // Get the name of the column from result table
269 const TString key = res.GetFieldName(i);
270
271 // Get the list with the allowed attributed from the map
272 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
273 if (!list)
274 return kFALSE;
275
276 // Check whether the current run (row[0]) with the current attribute
277 // (row[i]) and the current run-type (row[1]) matches the attribute
278 // of the last run (keys[i] with run-type row[i])
279 return list->Matches(keys[1], row[1], keys[i], row[i], atoi(row[0]));
280 }
281
282 Bool_t InsertSequence(Int_t from, Int_t to)
283 {
284 cout << " - Inserting Sequence into database." << endl;
285
286 // ========== Request number of events ==========
287
288 // Can be replaced by
289
290 const TString runtime =
291 "SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))), ";
292
293 const TString where = Form("(fRunNumber*1000+fFileNumber BETWEEN %d AND %d) "
294 "AND fTelescopeNumber=%d AND fExcludedFDAKEY=1",
295 from, to, fTelescopeNumber);
296
297 TString query;
298 query = "SELECT SUM(fNumEvents), ";
299 query += runtime;
300 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
301 query += " MIN(fAzimuth), MAX(fAzimuth), ";
302 query += " MIN(if(fRunStart=0,NULL,fRunStart)), ";
303 query += " MAX(if(fRunStop=0,NULL,fRunStop)), ";
304 query += " ELT(MAX(FIELD(fLightConditionsKEY, 1, 2, 7, 5, 8)), 1, 2, 7, 5, 8) ";
305 query += " FROM RunData WHERE ";
306 query += where;
307 query += " AND fRunTypeKEY=2";
308
309 TSQLResult *res = Query(query);
310 if (!res)
311 return kFALSE;
312
313 TSQLRow *row = res->Next();
314 if (!row || res->GetFieldCount()!=9)
315 {
316 cout << "ERROR - Wrong result from query: " << query << endl;
317 return kFALSE;
318 }
319
320 const TString nevts = (*row)[0];
321 const TString secs = (*row)[1];
322 const TString zdmin = (*row)[2];
323 const TString zdmax = (*row)[3];
324 const TString azmin = (*row)[4];
325 const TString azmax = (*row)[5];
326 const TString start = (*row)[6];
327 const TString stop = (*row)[7];
328 const TString light = (*row)[8];
329
330 delete res;
331
332 const TString elts = GetELTSource(where);
333 if (elts.IsNull())
334 return kFALSE;
335
336 const TString eltp = GetELTProject(where);
337 if (eltp.IsNull())
338 return kFALSE;
339
340 // ========== Request data of sequence ==========
341 query = "SELECT ";
342 query += elts;
343 query += ", ";
344 query += eltp;
345 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
346 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
347 " fTriggerDelayTableKEY, fObservationModeKEY, fSumTriggerFlagKEY "
348 " FROM RunData WHERE fRunTypeKEY=2 AND ";
349 query += where;
350 query += " LIMIT 1";
351
352 res = Query(query);
353 if (!res)
354 return kFALSE;
355
356 row = res->Next();
357 if (!row || res->GetFieldCount()!=9)
358 {
359 cout << "ERROR - No result from query: " << query << endl;
360 return kFALSE;
361 }
362
363 const TString set = Form("fSequenceFirst=%d, fTelescopeNumber=%d ", from/1000, fTelescopeNumber);
364
365 TString query1;
366 query1 += set;
367 query1 += Form(",fSequenceLast=%d,", to/1000);
368 query1 += Form(" fSourceKEY=%s,", (*row)[0]);
369 query1 += Form(" fProjectKEY=%s,", (*row)[1]);
370 query1 += Form(" fNumEvents=%s,", nevts.Data());
371 query1 += Form(" fRunTime=%s,", secs.Data());
372 query1 += Form(" fRunStart=\"%s\",", start.Data());
373 query1 += Form(" fRunStop=\"%s\",", stop.Data());
374 query1 += Form(" fZenithDistanceMin=%s,", zdmin.Data());
375 query1 += Form(" fZenithDistanceMax=%s,", zdmax.Data());
376 query1 += Form(" fAzimuthMin=%s,", azmin.Data());
377 query1 += Form(" fAzimuthMax=%s,", azmax.Data());
378 query1 += Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
379 query1 += Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
380 query1 += Form(" fHvSettingsKEY=%s,", (*row)[4]);
381 query1 += Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
382 query1 += Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
383 query1 += Form(" fLightConditionsKEY=%s,", light.Data());
384 query1 += Form(" fObservationModeKEY=%s, ", (*row)[7]);
385 query1 += Form(" fSumTriggerFlagKEY=%s, ", (*row)[8]);
386 query1 += "fManuallyChangedKEY=1";
387
388 delete res;
389
390 const TString where2 = Form("(fRunTypeKEY BETWEEN 2 AND 4) AND %s",
391 where.Data());
392
393 if (!Insert("Sequences", query1))
394 {
395 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
396 return kFALSE;
397 }
398
399 if (!Update("RunData", set, where))
400 {
401 cout << "ERROR - Could not update RunData." << endl;
402 return kFALSE;
403 }
404
405 TString prio = set;
406 prio += Form(", fPriority=%d ", from/1000);
407 if (!Insert("SequenceProcessStatus", prio))
408 {
409 cout << "ERROR - Could not insert Sequence into SequenceProcessStatus." << endl;
410 return kFALSE;
411 }
412
413 return kTRUE;
414 }
415
416 Bool_t DeleteSequence(Int_t sequ)
417 {
418 if (fPathRawData.IsNull() || fPathSequences.IsNull())
419 {
420 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
421 return kTRUE;
422 }
423
424 //queries to delete information from the database
425 const TString query(Form("fSequenceFirst=%d", sequ));
426
427 //commands to delete files from the disk
428 const TString fname(Form("%s/%04d/sequence%08d.txt", fPathSequences.Data(),sequ/10000, sequ));
429 const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
430 const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
431
432 if (!Delete("Calibration", query))
433 return 2;
434
435 if (!Delete("Star", query))
436 return 2;
437
438 if (!Delete("SequenceProcessStatus", query))
439 return 2;
440
441 if (!Delete("Sequences", query))
442 return 2;
443
444 if (!Update("RunData", "fSequenceFirst=0", query))
445 return 2;
446
447 if (IsDummy())
448 {
449 cout << " + would unlink " << fname << endl;
450 cout << " + would remove " << cmd1 << endl;
451 cout << " + would remove " << cmd2 << endl;
452 return kTRUE;
453 }
454
455 cout << " + will unlink " << fname << endl;
456 cout << " + will remove " << cmd1 << endl;
457 cout << " + will remove " << cmd2 << endl;
458
459 gSystem->Unlink(fname);
460
461 gSystem->Exec(cmd1);
462 gSystem->Exec(cmd2);
463
464 return kTRUE;
465 }
466
467 Int_t CheckSequence(Int_t runstart, Int_t runstop)
468 {
469 const char *fmt1 = "SELECT fRunNumber*1000+fFileNumber AS Id FROM RunData WHERE";
470 const char *fmt2 = "AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) ORDER BY Id";
471
472 const TString query1 = Form("%s fTelescopeNumber=%d AND fSequenceFirst=%d %s", fmt1, fTelescopeNumber, runstart/1000, fmt2);
473 const TString query2 = Form("%s fTelescopeNumber=%d AND fRunNumber*1000+fFileNumber BETWEEN %d AND %d %s", fmt1, fTelescopeNumber, runstart, runstop, fmt2);
474
475 TSQLResult *res1 = Query(query1);
476 if (!res1)
477 return 2;
478
479 TSQLResult *res2 = Query(query2);
480 if (!res2)
481 {
482 delete res1;
483 return 2;
484 }
485
486 while (1)
487 {
488 TSQLRow *row1 = res1->Next();
489 TSQLRow *row2 = res2->Next();
490
491 if (!row1 && !row2)
492 return kTRUE;
493
494 if (!row1 || !row2)
495 return kFALSE;
496
497 if (atoi((*row1)[0])!=atoi((*row2)[0]))
498 return kFALSE;
499 }
500
501 return kFALSE;
502 }
503
504 Int_t CreateSequence(Int_t runstart, Int_t runstop)
505 {
506 cout << " * Creating Sequence " << runstart << "-" << runstop << ":" << endl;
507
508 TString query=
509 Form("SELECT fSequenceFirst FROM RunData "
510 " WHERE fRunNumber*1000+fFileNumber BETWEEN %d AND %d AND "
511 " fTelescopeNumber=%d AND fSequenceFirst>0 AND "
512 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
513 " GROUP BY fSequenceFirst", runstart, runstop,
514 fTelescopeNumber);
515
516 TSQLResult *res = Query(query);
517 if (!res)
518 return 2;
519
520 const Int_t cnt = res->GetRowCount();
521
522 Int_t rc = kTRUE;
523 if (cnt==1)
524 {
525 TSQLRow *row=res->Next();
526 const Int_t check = CheckSequence(runstart, runstop);
527 if (check==kTRUE)
528 {
529 cout << " - Identical sequence already existing." << endl;
530 delete res;
531 return kTRUE;
532 }
533 if (check==2)
534 rc=2;
535 else
536 {
537 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
538 if (DeleteSequence(atoi((*row)[0]))==2)
539 rc = 2;
540 }
541 }
542 else
543 {
544 TSQLRow *row=0;
545 while ((row=res->Next()))
546 {
547 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
548 if (DeleteSequence(atoi((*row)[0]))==2)
549 rc = 2;
550 }
551 }
552
553 delete res;
554
555 if (rc==2)
556 return 2;
557
558 if (!InsertSequence(runstart, runstop))
559 return 2;
560
561 return kTRUE;
562 }
563
564 Bool_t ReadResources(const char *fname)
565 {
566 // Check for the section header
567 TPRegexp regexp("^\\[(Transition|Regexp):?[0-9 ]*\\]$");
568 // Check if section header contains a number
569 TPRegexp regnum("[0-9]");
570 // Check if section header contains the telescope number
571 TPRegexp regtel(Form("[^0-9]0*%d[^0-9]", fTelescopeNumber));
572
573 ifstream fin(fname);
574 if (!fin)
575 {
576 cout << "Cannot open file " << fname << ": ";
577 cout << strerror(errno) << endl;
578 return kFALSE;
579 }
580
581 Int_t section = 0;
582
583 TString key;
584 while (1)
585 {
586 TString txt;
587 txt.ReadLine(fin);
588 if (!fin)
589 break;
590
591 txt = txt.Strip(TString::kBoth);
592
593 if (txt[0]=='#' || txt.IsNull())
594 continue;
595
596 if (txt[0]=='[')
597 {
598 TString sec = txt(regexp);
599 if (!sec.IsNull())
600 {
601 // Skip sections with the wrong telescope number
602 // (If section contains numbers but not the tel num)
603 if (!sec(regnum).IsNull() && sec(regtel).IsNull())
604 continue;
605
606 // Check which section we are in
607 if (sec.BeginsWith("[Transition"))
608 section = 1;
609 if (sec.BeginsWith("[Regexp"))
610 section = 2;
611
612 continue;
613 }
614
615 if (section!=2)
616 {
617 cout << "WARNING - Line starts with [ but we are not in the Regexp section." << endl;
618 cout << txt << endl;
619 continue;
620 }
621 }
622
623
624 TObjArray *arr = txt.Tokenize(" ");
625
626 if (arr->GetEntries()>0)
627 switch (section)
628 {
629 case 1:
630 {
631 TString key = arr->At(0)->GetName();
632 key.Prepend("f");
633 key.Append("KEY");
634
635 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
636 if (!list)
637 {
638 //cout << key << endl;
639 list = new CheckList;
640 fMap.Add(new TObjString(key), list);
641 }
642
643 if (arr->GetEntries()>1)
644 {
645 //cout << key << " ";
646 list->Add(new CheckMatch(*arr, 1));
647 }
648 }
649 break;
650 case 2:
651 fListRegexp.Add(new Rule(*arr));
652 break;
653 }
654
655 delete arr;
656 }
657
658 return kTRUE;
659 }
660
661 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
662 {
663 TObjArray names; // array with old names (including regexp)
664
665 // Add to array and expand the array if necessary
666 TSQLRow *row=0;
667 while ((row=res->Next()))
668 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
669
670 // Now a LUT is build which converts the keys for
671 // the names including the regexp into keys for
672 // the names excluding the regexp
673 TString elt(Form("ELT(RunData.f%sKEY+1", col));
674
675 // loop over all entries in the list
676 const Int_t n = names.GetSize();
677 for (int i=0; i<n; i++)
678 {
679 // For all entries which are not in the list
680 // write an undefined value into the LUT
681 TObject *o = names.UncheckedAt(i);
682 if (!o)
683 {
684 elt += ",0";
685 continue;
686 }
687
688 // Remove the regexp from the string which includes it
689 TString name = o->GetName();
690
691 TIter NextR(&regexp);
692 TObject *obj=0;
693 while ((obj=NextR()))
694 {
695 TPRegexp reg(obj->GetName());
696 const Ssiz_t pos = name.Index(reg, 0);
697 if (pos>0)
698 {
699 name.Remove(pos);
700 name += "-W";
701 break;
702 }
703 }
704
705 // Check if such a Key exists, if not insert it
706 const Int_t key = QueryKeyOfName(col, name);
707
708 // RESOLVE return code!!!
709 //if (key<0)
710 // return "";
711
712 // add index to the LUT
713 elt += Form(",%d", key);
714 }
715
716 // close LUT expression
717 // elt += ") AS Elt";
718 // elt += col;
719
720 elt += ") AS f";
721 elt += col;
722 elt += "KEY";
723
724 // return result
725 return elt;
726 }
727
728 TString GetELT(const char *col, TSQLResult *res, TString regexp)
729 {
730 TList list;
731 list.SetOwner();
732 list.Add(new TObjString(regexp));
733 return GetELT(col, res, list);
734 }
735
736 TString GetELTQuery(const char *col, const char *cond) const
737 {
738 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
739 "LEFT JOIN %s USING (f%sKEY) WHERE %s GROUP BY f%sName",
740 col, col, col, col, cond, col);
741 }
742
743 TString GetELTSource(const char *cond)
744 {
745 //query all sources observed in this night
746 TSQLResult *resx = Query(GetELTQuery("Source", cond));
747 if (!resx)
748 return "";
749
750 // In the case there is only a single source
751 // do not replace the source key by the ELT
752 if (resx->GetRowCount()==1)
753 return "fSourceKEY";
754
755 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
756 delete resx;
757
758 return elts;
759 }
760
761 TString GetELTProject(const char *cond)
762 {
763 //query all project names observed in this night
764 TSQLResult *resx = Query(GetELTQuery("Project", cond));
765 if (!resx)
766 return "";
767
768 // In the case there is only a single project
769 // do not replace the project key by the ELT
770 if (resx->GetRowCount()==1)
771 return "fProjectKEY";
772
773 TList regexp;
774 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
775 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
776
777 TString eltp2 = GetELT("Project", resx, regexp);
778 delete resx;
779 regexp.Delete();
780
781 return eltp2;
782 }
783
784 Bool_t HasAtLeastOne(TString src, TString chk) const
785 {
786 src.ToLower();
787 chk.ToLower();
788
789 for (int i=0; i<chk.Length(); i++)
790 if (src.First(chk[i])<0)
791 return kFALSE;
792
793 return kTRUE;
794 }
795
796 TString PrepareString(TSQLResult &res, TArrayI &runs)
797 {
798 // Number of result rows
799 const Int_t rows = res.GetRowCount();
800
801 runs.Set(rows); // initialize size of array for run numbers
802
803 TArrayD start(rows); // instantiate array for start times
804 TArrayD stop(rows); // instantiate array for stop times
805
806 TString str; // result string
807
808 Int_t idx=0;
809 TSQLRow *row=0;
810 while ((row=res.Next()))
811 {
812 runs[idx] = atoi((*row)[0]); // run number
813
814 const TString tstart = ((*row)[2]); // start time
815 const TString tstop = ((*row)[3]); // stop time
816
817 start[idx] = MTime(tstart).GetMjd(); // convert to double
818 stop[idx] = MTime(tstop).GetMjd(); // convert to double
819
820 // This is a workaround for broken start-times
821 if (tstart=="0000-00-00 00:00:00")
822 start[idx] = stop[idx];
823
824 // Add a run-type character for this run to the string
825 str += RunType((*row)[1]);
826
827 // Increase index
828 idx++;
829 }
830
831 // Now the P- and D- runs are classified by the time-distance
832 // to the next or previous C-Run
833 Double_t lastc = -1;
834 Double_t nextc = -1;
835 for (int i=0; i<str.Length(); i++)
836 {
837 if (str[i]=='C')
838 {
839 // Remember stop time of C-Run as time of last C-run
840 lastc = stop[i];
841
842 // Calculate start time of next C-Run
843 const TString residual = str(i+1, str.Length());
844 const Int_t pos = residual.First('C');
845
846 // No C-Run found anymore. Finished...
847 if (pos<0)
848 break;
849
850 // Remember start time of C-Run as time of next C-run
851 nextc = start[i+1+pos];
852 continue;
853 }
854
855 // Check whether the identifying character should
856 // be converted to lower case
857 if (start[i]-lastc>nextc-stop[i])
858 str[i] = tolower(str[i]);
859 }
860 //cout << str << endl;
861 return str;
862 }
863
864 void PrintResidual(Int_t runstart, Int_t runstop, TString residual, const char *descr)
865 {
866 residual.ToLower();
867
868 // Count number of unsequences "characters"
869 const Int_t nump = residual.CountChar('p');
870 const Int_t numd = residual.CountChar('d');
871 const Int_t numc = residual.CountChar('c');
872
873 // Print some information to the output steram
874 if (nump+numc+numd==0)
875 return;
876
877 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
878 if (numd>0)
879 cout << " (DATA!)";
880
881 if (nump==0 || numc==0 || numd==0)
882 cout << " Missing";
883 if (numd==0)
884 cout << " D";
885 if (nump==0)
886 cout << " P";
887 if (numc==0)
888 cout << " C";
889
890 cout << endl;
891 }
892
893 Int_t SplitBlock(Int_t runstart, Int_t runstop, const TString &cond)
894 {
895 // Request data necessary to split block into sequences
896 /*const*/ TString query=
897 Form("SELECT fRunNumber*1000+fFileNumber AS Id, fRunTypeKEY, fRunStart, fRunStop"
898 " FROM RunData "
899 " WHERE fRunNumber*1000+fFileNumber BETWEEN %d AND %d AND "
900 /*" fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
901 " ORDER BY Id"*/, runstart, runstop);
902
903 query += cond;
904 query += " ORDER BY Id";
905
906
907 // Send query
908 TSQLResult *res = Query(query);
909 if (!res)
910 return 2;
911
912 // Get String containing the sequence of P-,C- and D-Runs
913 // and an array with the corresponding run-numbers
914 TArrayI runs;
915 const TString str = PrepareString(*res, runs);
916
917 delete res;
918
919 // Check if the prepared string at least contains one run of each type
920 if (!HasAtLeastOne(str, "PCD"))
921 {
922 PrintResidual(runstart, runstop, str, "Block");
923 return kTRUE;
924 }
925
926 // Start the sequence building
927 // ---------------------------
928
929 Int_t pos = 0; // Position in the string
930 TExMap map; // Map for resulting sequences
931 TString residual; // Unsequences "characters"
932
933 // Step through the string one character by one
934 cout << " ";
935 while (pos<str.Length())
936 {
937 Bool_t found = kFALSE;
938
939 // Loop over the predefined regular expressions
940 TIter Next(&fListRegexp);
941 Rule *obj = 0;
942 while ((obj=(Rule*)Next()))
943 {
944 // Check if regular expressions matchs
945 const Ssiz_t len = obj->Match(str, pos, runs[pos]);
946 if (len>0)
947 {
948 // In case of match get the substring which
949 // defines the sequence and print it
950 TString sub = str(pos, len);
951 sub.ToUpper();
952 cout << runs[pos]<<":"<< sub << "|";
953
954 // Add the first and last run of the sequence to the map
955 map.Add(runs[pos], runs[pos+len-1]);
956
957 // A sequence was found...
958 found = kTRUE;
959
960 // step forward in the string by the length of the sequence
961 pos += len;
962 break;
963 }
964 }
965
966 // If a sequence was found go on...
967 if (found)
968 continue;
969
970 // print unsequenced characters
971 cout << (char)tolower(str[pos]);
972
973 // Count the number of "characters" not sequenced
974 residual += str[pos];
975
976 // step one character forward
977 pos++;
978 }
979 cout << endl;
980
981 PrintResidual(runstart, runstop, residual, "Runs ");
982
983 // Create all sequences which were previously found
984 Long_t first, last;
985 TExMapIter iter(&map);
986 while (iter.Next(first, last))
987 if (CreateSequence(first, last)==2)
988 return 2;
989
990 return kTRUE;
991 }
992
993 Char_t RunType(Int_t n) const
994 {
995 switch (n)
996 {
997 case 2: return 'D';
998 case 3: return 'P';
999 case 4: return 'C';
1000 }
1001 return '-';
1002 }
1003 Char_t RunType(const char *str) const
1004 {
1005 return RunType(atoi(str));
1006 }
1007
1008 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks, const TString &cond)
1009 {
1010 // col key content
1011 // -----------------------------
1012 // 0 - runnumber
1013 // 1 0 RunTypeKEY
1014 // 2 1 source
1015 // 3 2 project
1016 // . .
1017 // . .
1018 // . . from transition.txt
1019 //
1020
1021 //build blocks of runs, which have the same values
1022 //for each block the first and the last run are stored in a TExMap
1023 //the values are checked with the help of an array of TStrings
1024 Long_t runstart = -1;
1025 Long_t runstop = -1;
1026
1027 const UInt_t ncols = res->GetFieldCount();
1028
1029 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
1030
1031 // Loop over runs
1032 TSQLRow *row=0;
1033 while ((row=res->Next()))
1034 {
1035 // This is the runnumber of the first run in the new block
1036 if (runstart<0)
1037 runstart=atoi((*row)[0]);
1038
1039 // Check which transitions might be allowed for this run and
1040 // which are not. Check also which attributes should be ignored
1041 // for this run. The information about it is stored in the map.
1042 Int_t rc[ncols];
1043 for (UInt_t i=2; i<ncols; i++)
1044 rc[i] = CheckTransition(*res, keys, *row, i);
1045
1046 for (UInt_t i=2; i<ncols; i++)
1047 {
1048 switch (rc[i])
1049 {
1050 case kTRUE: // Transition allowed, switch to new attribute
1051 keys[i] = (*row)[i];
1052 continue;
1053
1054 case -1: // Ignore attribute, continue with next one
1055 //if (keys[i] == (*row)[i])
1056 // break; // Only ignore it if it is really different
1057 continue;
1058
1059 case kFALSE: // No decision (nothing matching found in map)
1060 break; // go on checking
1061
1062 }
1063
1064 // If past-attribute is not yet initialized, do it now.
1065 if (keys[i].IsNull())
1066 keys[i] = (*row)[i];
1067
1068 // Check whether key has changed for this run
1069 // (this condition is never true for the first run)
1070 // This condition in only reached if keys[i] was initialized
1071 if (keys[i] == (*row)[i])
1072 continue;
1073
1074 // Found one block with unique keys, fill values into TExMap
1075 // (except if this is the first run processed)
1076 cout << endl;
1077 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1078 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1079 if (SplitBlock(runstart, runstop, cond)==2)
1080 return 2;
1081 cout << " - Transition from " << RunType(keys[1]) << ":";
1082 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1083 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1084 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1085
1086 // This is already the first run of the new block
1087 runstart=atoi((*row)[0]);
1088
1089 // These are the keys corresponding to the first run
1090 // in the new block. All following runs should have
1091 // identical keys. Do not set the attribute if for
1092 // this attribute the transition check evaluated
1093 // to "ignore".
1094 for (UInt_t j=2; j<ncols; j++)
1095 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1096
1097 break;
1098 }
1099
1100 keys[1] = (*row)[1]; // Remember run type for next run
1101
1102 // This is the new runnumber of the last run in this block
1103 runstop=atoi((*row)[0]);
1104 }
1105
1106 // If runstart==-1 a possible block would contain a single run...
1107 if (runstart>0 && runstart!=runstop)
1108 {
1109 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1110 //fill values into TExMap (last value)
1111 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1112 if (SplitBlock(runstart, runstop, cond)==2)
1113 return 2;
1114 }
1115 cout << "Done." << endl << endl;
1116
1117 return kTRUE;
1118 }
1119
1120public:
1121 SequenceBuild(Int_t tel=1, const char *rc="sql.rc") : MSQLMagic(rc), fTelescopeNumber(tel)
1122 {
1123 fListRegexp.SetOwner();
1124
1125 // FIXME: THIS IS NOT YET HANDLED
1126 if (ReadResources("resources/sequences.rc"))
1127 return;
1128 }
1129
1130 SequenceBuild(TEnv &env, Int_t tel=1) : MSQLMagic(env), fTelescopeNumber(tel)
1131 {
1132 fListRegexp.SetOwner();
1133
1134 // FIXME: THIS IS NOT YET HANDLED
1135 if (ReadResources("resources/sequences.rc"))
1136 return;
1137 }
1138 ~SequenceBuild()
1139 {
1140 fMap.DeleteAll();
1141 }
1142
1143 void SetPathRawData(const char *path) { fPathRawData=path; }
1144 void SetPathSequences(const char *path) { fPathSequences=path; }
1145
1146 int Build(TString day)
1147 {
1148 cout << endl;
1149 cout << "Night of sunrise at " << day << ":" << endl;
1150
1151 day += " 13:00:00";
1152 const TString cond =
1153 Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\") "
1154 "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 "
1155 "AND fTelescopeNumber=%d ",
1156 day.Data(), day.Data(), fTelescopeNumber);
1157
1158 //query all sources observed in this night
1159 const TString elts = GetELTSource(cond);
1160 if (elts.IsNull())
1161 return 2;
1162
1163 //query all project names observed in this night
1164 const TString eltp2 = GetELTProject(cond);
1165 if (elts.IsNull())
1166 return 2;
1167
1168 // Setup query to get all values from the database,
1169 // that are relevant for building sequences
1170 TString query("SELECT fRunNumber*1000+fFileNumber AS Id, fRunTypeKEY, ");
1171 query += elts;
1172 query += ", ";
1173 query += eltp2;
1174
1175 // Now add all entries from the transition table to the query
1176 TIter NextPair(&fMap);
1177 TObject *mapkey = 0;
1178 while ((mapkey=(TPair*)NextPair()))
1179 if (!query.Contains(mapkey->GetName()))
1180 {
1181 query += ", ";
1182 query += mapkey->GetName();
1183 }
1184 query += Form(" FROM RunData WHERE %s ORDER BY Id", cond.Data());
1185
1186 TSQLResult *res = Query(query);
1187 if (!res)
1188 return 2;
1189
1190 TExMap blocks;
1191 const Int_t rc = BuildBlocks(res, blocks, cond);
1192 delete res;
1193
1194 return rc;
1195 }
1196
1197 Int_t Build()
1198 {
1199 //get all dates from the database
1200 TSQLResult *res = Query("SELECT fDate FROM SequenceBuildStatus ORDER BY fDate DESC");
1201 if (!res)
1202 return 2;
1203
1204 //execute buildsequenceentries for all dates
1205 TSQLRow *row=0;
1206 while ((row=res->Next()))
1207 Build((*row)[0]);
1208
1209 delete res;
1210
1211 return 1;
1212 }
1213 ClassDef(SequenceBuild, 0)
1214};
1215
1216ClassImp(SequenceBuild);
1217
1218int buildsequenceentries(TString day, TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1219{
1220 SequenceBuild serv(tel, "sql.rc");
1221 if (!serv.IsConnected())
1222 {
1223 cout << "ERROR - Connection to database failed." << endl;
1224 return 0;
1225 }
1226
1227 cout << "buildsequenceentries" << endl;
1228 cout << "--------------------" << endl;
1229 cout << endl;
1230 cout << "Connected to " << serv.GetName() << endl;
1231 if (!datapath.IsNull())
1232 cout << "DataPath: " << datapath << endl;
1233 if (!sequpath.IsNull())
1234 cout << "SeqPath: " << sequpath << endl;
1235 cout << "Day: " << day << endl;
1236 cout << "Telescope: " << tel << endl;
1237 cout << endl;
1238
1239 serv.SetIsDummy(dummy);
1240 serv.SetPathRawData(datapath);
1241 serv.SetPathSequences(sequpath);
1242 return serv.Build(day);
1243}
1244
1245//
1246// Build Sequences for all Nights
1247//
1248int buildsequenceentries(TString datapath, TString sequpath, Int_t tel=1, Bool_t dummy=kTRUE)
1249{
1250 SequenceBuild serv(tel, "sql.rc");
1251 if (!serv.IsConnected())
1252 {
1253 cout << "ERROR - Connection to database failed." << endl;
1254 return 0;
1255 }
1256
1257 cout << "buildsequenceentries" << endl;
1258 cout << "--------------------" << endl;
1259 cout << endl;
1260 cout << "Connected to " << serv.GetName() << endl;
1261 cout << "DataPath: " << datapath << endl;
1262 cout << "SeqPath: " << sequpath << endl;
1263 cout << "Telescope: " << tel << endl;
1264 cout << endl;
1265
1266 serv.SetIsDummy(dummy);
1267 serv.SetPathRawData(datapath);
1268 serv.SetPathSequences(sequpath);
1269 return serv.Build();
1270}
1271
1272int buildsequenceentries(Int_t tel=1, Bool_t dummy=kTRUE)
1273{
1274 return buildsequenceentries("", "", tel, dummy);
1275}
1276
1277int buildsequenceentries(TString day, Int_t tel=1, Bool_t dummy=kTRUE)
1278{
1279 return buildsequenceentries(day, "", "", tel, dummy);
1280}
Note: See TracBrowser for help on using the repository browser.