source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 8987

Last change on this file since 8987 was 8968, checked in by tbretz, 17 years ago
*** empty log message ***
File size: 37.8 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2006
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82#include <errno.h>
83
84#include <TSQLRow.h>
85#include <TSQLResult.h>
86
87#include <TEnv.h>
88#include <TMap.h>
89#include <TMath.h>
90#include <TExMap.h>
91#include <TArrayI.h>
92#include <TArrayD.h>
93#include <TPRegexp.h>
94#include <TSystem.h>
95#include <TObjString.h>
96#include <TObjArray.h>
97
98#include "MTime.h"
99#include "MDirIter.h"
100
101#include "MSQLMagic.h"
102
103using namespace std;
104
105
106class Rule : public TObject
107{
108private:
109 TPRegexp fRegexp;
110
111 UInt_t fMin;
112 UInt_t fMax;
113
114public:
115 Rule(const char *regexp, UInt_t min=0, UInt_t max=(UInt_t)-1) :
116 fRegexp(Form("^%s", regexp)), fMin(min), fMax(max)
117 {
118 }
119
120 Rule(TObjArray &arr) :
121 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
122 fMin(0), fMax((UInt_t)-1)
123 {
124 if (arr.GetEntries()>1)
125 fMin = atoi(arr[1]->GetName());
126 if (arr.GetEntries()>2)
127 fMax = atoi(arr[2]->GetName());
128
129 //cout << "Regexp: " << (arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):"") << " " << fMin << " " << fMax << endl;
130 }
131
132 Ssiz_t Match(const TString &str, Int_t idx, Int_t run=-1)
133 {
134 if (!IsValid(run))
135 return 0;
136
137 TString mods;
138 TArrayI pos;
139 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
140
141 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
142 }
143
144 Bool_t IsValid(UInt_t run) const { return run<0 || (run>=fMin && run<=fMax); }
145 ClassDef(Rule, 0)
146};
147ClassImp(Rule);
148
149class CheckMatch : public TObject
150{
151private:
152 TPRegexp fRunType1;
153 TPRegexp fRunType2;
154
155 TPRegexp fRegexp1;
156 TPRegexp fRegexp2;
157
158 UInt_t fMin;
159 UInt_t fMax;
160
161 void Init(const TObjArray &arr, Int_t idx=0)
162 {
163 const Int_t n = arr.GetEntries();
164
165 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
166
167 for (int i=0; i<n-idx; i++)
168 {
169 //cout << arr[i+idx]->GetName() << " ";
170
171 TString str(arr[i+idx]->GetName());
172 if (str=="*")
173 str = ".*";
174
175 switch (isminus && i>1 ? i+1 : i)
176 {
177 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
178 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
179 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
180 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
181 case 4: fMin = str.Atoi(); break;
182 case 5: fMax = str.Atoi(); break;
183 }
184 }
185 //cout << endl;
186 }
187
188public:
189 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
190
191 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
192 {
193 TObjArray *arr = txt.Tokenize(" ");
194 Init(*arr);
195 delete arr;
196 }
197
198 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
199 {
200 Init(arr, idx);
201 }
202
203 CheckMatch(const char *from, const char *to, Int_t min=0, Int_t max=-1)
204 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
205 {
206 }
207
208 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, UInt_t run=0)
209 {
210 if (run>0)
211 {
212 if (run<fMin || run>fMax)
213 return kFALSE;
214 }
215
216 const TString test("X-"); // FIXME:STUPID!
217
218 //if (test.Index(fRunType2)==1)
219 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
220
221 if (test.Index(fRunType1,0)==1)
222 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
223
224 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
225 }
226 ClassDef(CheckMatch,0)
227};
228ClassImp(CheckMatch);
229
230class CheckList : public TList
231{
232public:
233 CheckList() { SetOwner(); }
234 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Int_t run=-1) const
235 {
236 TIter Next(this);
237
238 CheckMatch *check = 0;
239
240 while ((check=(CheckMatch*)Next()))
241 {
242 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
243 if (rc)
244 return rc;
245 }
246
247 return kFALSE;
248 }
249 ClassDef(CheckList,0)
250};
251ClassImp(CheckList);
252
253class SequenceBuild : public MSQLMagic
254{
255private:
256 TString fPathRawData;
257 TString fPathSequences;
258
259 TMap fMap;
260 TList fListRegexp;
261
262 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
263 {
264 // Get the name of the column from result table
265 const TString key = res.GetFieldName(i);
266
267 // Get the list with the allowed attributed from the map
268 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
269 if (!list)
270 return kFALSE;
271
272 // Check whether the current run (row[0]) with the current attribute
273 // (row[i]) and the current run-type (row[1]) matches the attribute
274 // of the last run (keys[i] with run-type row[i])
275 return list->Matches(keys[1], row[1], keys[i], row[i], atoi(row[0]));
276 }
277
278 Bool_t InsertSequence(Int_t from, Int_t to)
279 {
280 cout << " - Inserting Sequence into database." << endl;
281
282 // ========== Request number of events ==========
283
284 // Can be replaced by
285
286 const TString runtime =
287 "SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))), ";
288 //"SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
289 //" TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
290 //" TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ";
291
292 const TString where = Form("(fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
293 from, to);
294
295 TString query;
296 query = "SELECT SUM(fNumEvents), ";
297 query += runtime;
298 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
299 query += " MIN(fAzimuth), MAX(fAzimuth), ";
300 query += " MIN(if(fRunStart=0,NULL,fRunStart)), ";
301 query += " MAX(if(fRunStop=0,NULL,fRunStop)), ";
302 query += " ELT(MAX(FIELD(fLightConditionsKEY, 1, 2, 7, 5, 8)), 1, 2, 7, 5, 8) ";
303 query += " FROM RunData WHERE ";
304 query += where;
305 query += " AND fRunTypeKEY=2";
306
307 TSQLResult *res = Query(query);
308 if (!res)
309 return kFALSE;
310
311 TSQLRow *row = res->Next();
312 if (!row || res->GetFieldCount()!=9)
313 {
314 cout << "ERROR - Wrong result from query: " << query << endl;
315 return kFALSE;
316 }
317
318 const TString nevts = (*row)[0];
319 const TString secs = (*row)[1];
320 const TString zdmin = (*row)[2];
321 const TString zdmax = (*row)[3];
322 const TString azmin = (*row)[4];
323 const TString azmax = (*row)[5];
324 const TString start = (*row)[6];
325 const TString stop = (*row)[7];
326 const TString light = (*row)[8];
327
328 delete res;
329
330 const TString elts = GetELTSource(where);
331 if (elts.IsNull())
332 return kFALSE;
333
334 const TString eltp = GetELTProject(where);
335 if (eltp.IsNull())
336 return kFALSE;
337
338 // ========== Request data of sequence ==========
339 query = "SELECT ";
340 query += elts;
341 query += ", ";
342 query += eltp;
343 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
344 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
345 " fTriggerDelayTableKEY, fObservationModeKEY "
346 " FROM RunData WHERE fRunTypeKEY=2 AND ";
347 query += where;
348 query += " LIMIT 1";
349
350 res = Query(query);
351 if (!res)
352 return kFALSE;
353
354 row = res->Next();
355 if (!row || res->GetFieldCount()!=8)
356 {
357 cout << "ERROR - No result from query: " << query << endl;
358 return kFALSE;
359 }
360
361 const TString set = Form("fSequenceFirst=%d ", from);
362
363 TString query1;
364 query1 += set;
365 query1 += Form(",fSequenceLast=%d,", to);
366 query1 += Form(" fSourceKEY=%s,", (*row)[0]);
367 query1 += Form(" fProjectKEY=%s,", (*row)[1]);
368 query1 += Form(" fNumEvents=%s,", nevts.Data());
369 query1 += Form(" fRunTime=%s,", secs.Data());
370 query1 += Form(" fRunStart=\"%s\",", start.Data());
371 query1 += Form(" fRunStop=\"%s\",", stop.Data());
372 query1 += Form(" fZenithDistanceMin=%s,", zdmin.Data());
373 query1 += Form(" fZenithDistanceMax=%s,", zdmax.Data());
374 query1 += Form(" fAzimuthMin=%s,", azmin.Data());
375 query1 += Form(" fAzimuthMax=%s,", azmax.Data());
376 query1 += Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
377 query1 += Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
378 query1 += Form(" fHvSettingsKEY=%s,", (*row)[4]);
379 query1 += Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
380 query1 += Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
381 query1 += Form(" fLightConditionsKEY=%s,", light.Data());
382 query1 += Form(" fObservationModeKEY=%s, ", (*row)[7]);
383 query1 += "fManuallyChangedKEY=1";
384
385 delete res;
386
387 const TString where2 = Form("(fRunTypeKEY BETWEEN 2 AND 4) AND %s",
388 where.Data());
389
390 if (!Insert("Sequences", query1))
391 {
392 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
393 return kFALSE;
394 }
395
396 if (!Update("RunData", set, where))
397 {
398 cout << "ERROR - Could not update RunData." << endl;
399 return kFALSE;
400 }
401
402 if (!Insert("SequenceProcessStatus", set))
403 {
404 cout << "ERROR - Could not insert Sequence into SequenceProcessStatus." << endl;
405 return kFALSE;
406 }
407
408 return kTRUE;
409 }
410
411 Bool_t DeleteSequence(Int_t sequ)
412 {
413 if (fPathRawData.IsNull() || fPathSequences.IsNull())
414 {
415 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
416 return kTRUE;
417 }
418
419 //queries to delete information from the database
420 const TString query(Form("fSequenceFirst=%d", sequ));
421
422 //commands to delete files from the disk
423 const TString fname(Form("%s/%04d/sequence%08d.txt", fPathSequences.Data(),sequ/10000, sequ));
424 const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
425 const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
426
427 if (!Delete("Calibration", query))
428 return 2;
429
430 if (!Delete("Star", query))
431 return 2;
432
433 if (!Delete("SequenceProcessStatus", query))
434 return 2;
435
436 if (!Delete("Sequences", query))
437 return 2;
438
439 if (!Update("RunData", "fSequenceFirst=0", query))
440 return 2;
441
442 if (IsDummy())
443 {
444 cout << " + unlink " << fname << endl;
445 cout << " + " << cmd1 << endl;
446 cout << " + " << cmd2 << endl;
447 return kTRUE;
448 }
449
450 gSystem->Unlink(fname);
451
452 gSystem->Exec(cmd1);
453 gSystem->Exec(cmd2);
454
455 return kTRUE;
456 }
457
458 Int_t CheckSequence(Int_t runstart, Int_t runstop)
459 {
460 const char *fmt1 = "SELECT fRunNumber FROM RunData WHERE";
461 const char *fmt2 = "AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) ORDER BY fRunNumber";
462
463 const TString query1 = Form("%s fSequenceFirst=%d %s", fmt1, runstart, fmt2);
464 const TString query2 = Form("%s fRunNumber BETWEEN %d AND %d %s", fmt1, runstart, runstop, fmt2);
465
466 TSQLResult *res1 = Query(query1);
467 if (!res1)
468 return 2;
469
470 TSQLResult *res2 = Query(query2);
471 if (!res2)
472 {
473 delete res1;
474 return 2;
475 }
476
477 while (1)
478 {
479 TSQLRow *row1 = res1->Next();
480 TSQLRow *row2 = res2->Next();
481
482 if (!row1 && !row2)
483 return kTRUE;
484
485 if (!row1 || !row2)
486 return kFALSE;
487
488 if (atoi((*row1)[0])!=atoi((*row2)[0]))
489 return kFALSE;
490 }
491
492 return kFALSE;
493 }
494
495 Int_t CreateSequence(Int_t runstart, Int_t runstop)
496 {
497 cout << " * Creating Sequence " << runstart << "-" << runstop << ":" << endl;
498
499 TString query=
500 Form("SELECT fSequenceFirst FROM RunData "
501 " WHERE fRunNumber BETWEEN %d AND %d AND "
502 " fSequenceFirst>0 AND "
503 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
504 " GROUP BY fSequenceFirst", runstart, runstop);
505
506 TSQLResult *res = Query(query);
507 if (!res)
508 return 2;
509
510 const Int_t cnt = res->GetRowCount();
511
512 Int_t rc = kTRUE;
513 if (cnt==1)
514 {
515 TSQLRow *row=res->Next();
516 const Int_t check = CheckSequence(runstart, runstop);
517 if (check==kTRUE)
518 {
519 cout << " - Identical sequence already existing." << endl;
520 delete res;
521 return kTRUE;
522 }
523 if (check==2)
524 rc=2;
525 else
526 {
527 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
528 if (DeleteSequence(atoi((*row)[0]))==2)
529 rc = 2;
530 }
531 }
532 else
533 {
534 TSQLRow *row=0;
535 while ((row=res->Next()))
536 {
537 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
538 if (DeleteSequence(atoi((*row)[0]))==2)
539 rc = 2;
540 }
541 }
542
543 delete res;
544
545 if (rc==2)
546 return 2;
547
548 if (!InsertSequence(runstart, runstop))
549 return 2;
550
551 return kTRUE;
552 }
553
554 Bool_t ReadResources(const char *fname)
555 {
556 TPRegexp regexp("^\\[.*\\]$");
557
558 ifstream fin(fname);
559 if (!fin)
560 {
561 cout << "Cannot open file " << fname << ": ";
562 cout << strerror(errno) << endl;
563 return kFALSE;
564 }
565
566 Int_t section = 0;
567
568 TString key;
569 while (1)
570 {
571 TString txt;
572 txt.ReadLine(fin);
573 if (!fin)
574 break;
575
576 txt = txt.Strip(TString::kBoth);
577
578 if (txt[0]=='#' || txt.IsNull())
579 continue;
580
581 if (txt[0]=='[' && section!=2)
582 {
583 //cout << txt << endl;
584 section = 0;
585 if (txt(regexp)=="[Transition]")
586 section = 1;
587 if (txt(regexp)=="[Regexp]")
588 section = 2;
589 continue;
590 }
591
592 TObjArray *arr = txt.Tokenize(" ");
593
594 if (arr->GetEntries()>0)
595 switch (section)
596 {
597 case 1:
598 {
599 TString key = arr->At(0)->GetName();
600 key.Prepend("f");
601 key.Append("KEY");
602
603 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
604 if (!list)
605 {
606 //cout << key << endl;
607 list = new CheckList;
608 fMap.Add(new TObjString(key), list);
609 }
610
611 if (arr->GetEntries()>1)
612 {
613 //cout << key << " ";
614 list->Add(new CheckMatch(*arr, 1));
615 }
616 }
617 break;
618 case 2:
619 fListRegexp.Add(new Rule(*arr));
620 break;
621 }
622
623 delete arr;
624 }
625
626 return kTRUE;
627 }
628
629 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
630 {
631 TObjArray names; // array with old names (including regexp)
632
633 // Add to array and expand the array if necessary
634 TSQLRow *row=0;
635 while ((row=res->Next()))
636 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
637
638 // Now a LUT is build which converts the keys for
639 // the names including the regexp into keys for
640 // the names excluding the regexp
641 TString elt(Form("ELT(RunData.f%sKEY+1", col));
642
643 // loop over all entries in the list
644 const Int_t n = names.GetSize();
645 for (int i=0; i<n; i++)
646 {
647 // For all entries which are not in the list
648 // write an undefined value into the LUT
649 TObject *o = names.UncheckedAt(i);
650 if (!o)
651 {
652 elt += ",0";
653 continue;
654 }
655
656 // Remove the regexp from the string which includes it
657 TString name = o->GetName();
658
659 TIter NextR(&regexp);
660 TObject *obj=0;
661 while ((obj=NextR()))
662 {
663 TPRegexp reg(obj->GetName());
664 const Ssiz_t pos = name.Index(reg, 0);
665 if (pos>0)
666 {
667 name.Remove(pos);
668 name += "-W";
669 break;
670 }
671 }
672
673 // Check if such a Key exists, if not insert it
674 const Int_t key = QueryKeyOfName(col, name);
675
676 // RESOLVE return code!!!
677 //if (key<0)
678 // return "";
679
680 // add index to the LUT
681 elt += Form(",%d", key);
682 }
683
684 // close LUT expression
685 // elt += ") AS Elt";
686 // elt += col;
687
688 elt += ") AS f";
689 elt += col;
690 elt += "KEY";
691
692 // return result
693 return elt;
694 }
695
696 TString GetELT(const char *col, TSQLResult *res, TString regexp)
697 {
698 TList list;
699 list.SetOwner();
700 list.Add(new TObjString(regexp));
701 return GetELT(col, res, list);
702 }
703
704 TString GetELTQuery(const char *col, const char *cond) const
705 {
706 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
707 "LEFT JOIN %s ON RunData.f%sKEY=%s.f%sKEY "
708 "WHERE %s GROUP BY f%sName",
709 col, col, col, col, col, col, cond, col);
710 }
711
712 TString GetELTSource(const char *cond)
713 {
714 //query all sources observed in this night
715 TSQLResult *resx = Query(GetELTQuery("Source", cond));
716 if (!resx)
717 return "";
718
719 // In the case there is only a single source
720 // do not replace the source key by the ELT
721 if (resx->GetRowCount()==1)
722 return "fSourceKEY";
723
724 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
725 delete resx;
726
727 return elts;
728 }
729
730 TString GetELTProject(const char *cond)
731 {
732 //query all project names observed in this night
733 TSQLResult *resx = Query(GetELTQuery("Project", cond));
734 if (!resx)
735 return "";
736
737 // In the case there is only a single project
738 // do not replace the project key by the ELT
739 if (resx->GetRowCount()==1)
740 return "fProjectKEY";
741
742 TList regexp;
743 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
744 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
745
746 TString eltp2 = GetELT("Project", resx, regexp);
747 delete resx;
748 regexp.Delete();
749
750 return eltp2;
751 }
752
753 Bool_t HasAtLeastOne(TString src, TString chk) const
754 {
755 src.ToLower();
756 chk.ToLower();
757
758 for (int i=0; i<chk.Length(); i++)
759 if (src.First(chk[i])<0)
760 return kFALSE;
761
762 return kTRUE;
763 }
764
765 TString PrepareString(TSQLResult &res, TArrayI &runs)
766 {
767 // Number of result rows
768 const Int_t rows = res.GetRowCount();
769
770 runs.Set(rows); // initialize size of array for run numbers
771
772 TArrayD start(rows); // instantiate array for start times
773 TArrayD stop(rows); // instantiate array for stop times
774
775 TString str; // result string
776
777 Int_t idx=0;
778 TSQLRow *row=0;
779 while ((row=res.Next()))
780 {
781 runs[idx] = atoi((*row)[0]); // run number
782
783 const TString tstart = ((*row)[2]); // start time
784 const TString tstop = ((*row)[3]); // stop time
785
786 start[idx] = MTime(tstart).GetMjd(); // convert to double
787 stop[idx] = MTime(tstop).GetMjd(); // convert to double
788
789 // This is a workaround for broken start-times
790 if (tstart=="0000-00-00 00:00:00")
791 start[idx] = stop[idx];
792
793 // Add a run-type character for this run to the string
794 str += RunType((*row)[1]);
795
796 // Increase index
797 idx++;
798 }
799
800 // Now the P- and D- runs are classified by the time-distance
801 // to the next or previous C-Run
802 Double_t lastc = -1;
803 Double_t nextc = -1;
804 for (int i=0; i<str.Length(); i++)
805 {
806 if (str[i]=='C')
807 {
808 // Remember stop time of C-Run as time of last C-run
809 lastc = stop[i];
810
811 // Calculate start time of next C-Run
812 const TString residual = str(i+1, str.Length());
813 const Int_t pos = residual.First('C');
814
815 // No C-Run found anymore. Finished...
816 if (pos<0)
817 break;
818
819 // Remember start time of C-Run as time of next C-run
820 nextc = start[i+1+pos];
821 continue;
822 }
823
824 // Check whether the identifying character should
825 // be converted to lower case
826 if (start[i]-lastc>nextc-stop[i])
827 str[i] = tolower(str[i]);
828 }
829 //cout << str << endl;
830 return str;
831 }
832
833 void PrintResidual(Int_t runstart, Int_t runstop, TString residual, const char *descr)
834 {
835 residual.ToLower();
836
837 // Count number of unsequences "characters"
838 const Int_t nump = residual.CountChar('p');
839 const Int_t numd = residual.CountChar('d');
840 const Int_t numc = residual.CountChar('c');
841
842 // Print some information to the output steram
843 if (nump+numc+numd==0)
844 return;
845
846 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
847 if (numd>0)
848 cout << " (DATA!)";
849
850 if (nump==0 || numc==0 || numd==0)
851 cout << " Missing";
852 if (numd==0)
853 cout << " D";
854 if (nump==0)
855 cout << " P";
856 if (numc==0)
857 cout << " C";
858
859 cout << endl;
860 }
861
862 Int_t SplitBlock(Int_t runstart, Int_t runstop)
863 {
864 // Request data necessary to split block into sequences
865 const TString query=
866 Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
867 " FROM RunData "
868 " WHERE fRunNumber BETWEEN %d AND %d AND "
869 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
870 " ORDER BY fRunNumber", runstart, runstop);
871
872 // Send query
873 TSQLResult *res = Query(query);
874 if (!res)
875 return 2;
876
877 // Get String containing the sequence of P-,C- and D-Runs
878 // and an array with the corresponding run-numbers
879 TArrayI runs;
880 const TString str = PrepareString(*res, runs);
881
882 delete res;
883
884 // Check if the prepared string at least contains one run of each type
885 if (!HasAtLeastOne(str, "PCD"))
886 {
887 PrintResidual(runstart, runstop, str, "Block");
888 return kTRUE;
889 }
890
891 // Start the sequence building
892 // ---------------------------
893
894 Int_t pos = 0; // Position in the string
895 TExMap map; // Map for resulting sequences
896 TString residual; // Unsequences "characters"
897
898 // Step through the string one character by one
899 cout << " ";
900 while (pos<str.Length())
901 {
902 Bool_t found = kFALSE;
903
904 // Loop over the predefined regular expressions
905 TIter Next(&fListRegexp);
906 Rule *obj = 0;
907 while ((obj=(Rule*)Next()))
908 {
909 // Check if regular expressions matchs
910 const Ssiz_t len = obj->Match(str, pos, runs[pos]);
911 if (len>0)
912 {
913 // In case of match get the substring which
914 // defines the sequence and print it
915 TString sub = str(pos, len);
916 sub.ToUpper();
917 cout << runs[pos]<<":"<< sub << "|";
918
919 // Add the first and last run of the sequence to the map
920 map.Add(runs[pos], runs[pos+len-1]);
921
922 // A sequence was found...
923 found = kTRUE;
924
925 // step forward in the string by the length of the sequence
926 pos += len;
927 break;
928 }
929 }
930
931 // If a sequence was found go on...
932 if (found)
933 continue;
934
935 // print unsequenced characters
936 cout << (char)tolower(str[pos]);
937
938 // Count the number of "characters" not sequenced
939 residual += str[pos];
940
941 // step one character forward
942 pos++;
943 }
944 cout << endl;
945
946 PrintResidual(runstart, runstop, residual, "Runs ");
947
948 // Create all sequences which were previously found
949 Long_t first, last;
950 TExMapIter iter(&map);
951 while (iter.Next(first, last))
952 if (CreateSequence(first, last)==2)
953 return 2;
954
955 return kTRUE;
956 }
957
958 Char_t RunType(Int_t n) const
959 {
960 switch (n)
961 {
962 case 2: return 'D';
963 case 3: return 'P';
964 case 4: return 'C';
965 }
966 return '-';
967 }
968 Char_t RunType(const char *str) const
969 {
970 return RunType(atoi(str));
971 }
972
973 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks)
974 {
975 // col key content
976 // -----------------------------
977 // 0 - runnumber
978 // 1 0 RunTypeKEY
979 // 2 1 source
980 // 3 2 project
981 // . .
982 // . .
983 // . . from transition.txt
984 //
985
986 //build blocks of runs, which have the same values
987 //for each block the first and the last run are stored in a TExMap
988 //the values are checked with the help of an array of TStrings
989 Long_t runstart = -1;
990 Long_t runstop = -1;
991
992 const UInt_t ncols = res->GetFieldCount();
993
994 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
995
996 // Loop over runs
997 TSQLRow *row=0;
998 while ((row=res->Next()))
999 {
1000 // This is the runnumber of the first run in the new block
1001 if (runstart<0)
1002 runstart=atoi((*row)[0]);
1003
1004 // Check which transitions might be allowed for this run and
1005 // which are not. Check also which attributes should be ignored
1006 // for this run. The information about it is stored in the map.
1007 Int_t rc[ncols];
1008 for (UInt_t i=2; i<ncols; i++)
1009 rc[i] = CheckTransition(*res, keys, *row, i);
1010
1011 for (UInt_t i=2; i<ncols; i++)
1012 {
1013 switch (rc[i])
1014 {
1015 case kTRUE: // Transition allowed, switch to new attribute
1016 keys[i] = (*row)[i];
1017 continue;
1018
1019 case -1: // Ignore attribute, continue with next one
1020 //if (keys[i] == (*row)[i])
1021 // break; // Only ignore it if it is really different
1022 continue;
1023
1024 case kFALSE: // No decision (nothing matching found in map)
1025 break; // go on checking
1026
1027 }
1028
1029 // If past-attribute is not yet initialized, do it now.
1030 if (keys[i].IsNull())
1031 keys[i] = (*row)[i];
1032
1033 // Check whether key has changed for this run
1034 // (this condition is never true for the first run)
1035 // This condition in only reached if keys[i] was initialized
1036 if (keys[i] == (*row)[i])
1037 continue;
1038
1039 // Found one block with unique keys, fill values into TExMap
1040 // (except if this is the first run processed)
1041 cout << endl;
1042 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1043 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1044 if (SplitBlock(runstart, runstop)==2)
1045 return 2;
1046 cout << " - Transition from " << RunType(keys[1]) << ":";
1047 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1048 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1049 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1050
1051 // This is already the first run of the new block
1052 runstart=atoi((*row)[0]);
1053
1054 // These are the keys corresponding to the first run
1055 // in the new block. All following runs should have
1056 // identical keys. Do not set the attribute if for
1057 // this attribute the transition check evaluated
1058 // to "ignore".
1059 for (UInt_t j=2; j<ncols; j++)
1060 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1061
1062 break;
1063 }
1064
1065 keys[1] = (*row)[1]; // Remember run type for next run
1066
1067 // This is the new runnumber of the last run in this block
1068 runstop=atoi((*row)[0]);
1069 }
1070
1071 // If runstart==-1 a possible block would contain a single run...
1072 if (runstart>0 && runstart!=runstop)
1073 {
1074 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1075 //fill values into TExMap (last value)
1076 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1077 if (SplitBlock(runstart, runstop)==2)
1078 return 2;
1079 }
1080 cout << "Done." << endl << endl;
1081
1082 return kTRUE;
1083 }
1084
1085public:
1086 SequenceBuild(TEnv &env) : MSQLMagic(env)
1087 {
1088 cout << "buildsequences" << endl;
1089 cout << "--------------" << endl;
1090 cout << endl;
1091 cout << "Connected to " << GetName() << endl;
1092
1093 fListRegexp.SetOwner();
1094
1095 // FIXME: THIS IS NOT YET HANDLED
1096 if (ReadResources("resources/sequences.rc"))
1097 return;
1098 }
1099 SequenceBuild()
1100 {
1101 fMap.DeleteAll();
1102 }
1103
1104 void SetPathRawData(const char *path) { fPathRawData=path; }
1105 void SetPathSequences(const char *path) { fPathSequences=path; }
1106
1107 int Build(TString day)
1108 {
1109 cout << endl;
1110 cout << "Night of sunrise at " << day << ":" << endl;
1111
1112 day += " 13:00:00";
1113 const TString cond =
1114 Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\") "
1115 "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 ",
1116 day.Data(), day.Data());
1117
1118 //query all sources observed in this night
1119 const TString elts = GetELTSource(cond);
1120 if (elts.IsNull())
1121 return 2;
1122
1123 //query all project names observed in this night
1124 const TString eltp2 = GetELTProject(cond);
1125 if (elts.IsNull())
1126 return 2;
1127
1128 // Setup query to get all values from the database,
1129 // that are relevant for building sequences
1130 TString query("SELECT fRunNumber, fRunTypeKEY, ");
1131 query += elts;
1132 query += ", ";
1133 query += eltp2;
1134
1135 // Now add all entries from the transition table to the query
1136 TIter NextPair(&fMap);
1137 TObject *mapkey = 0;
1138 while ((mapkey=(TPair*)NextPair()))
1139 if (!query.Contains(mapkey->GetName()))
1140 {
1141 query += ", ";
1142 query += mapkey->GetName();
1143 }
1144 query += Form(" FROM RunData WHERE %s ORDER BY fRunNumber", cond.Data());
1145
1146 TSQLResult *res = Query(query);
1147 if (!res)
1148 return 2;
1149
1150 TExMap blocks;
1151 const Int_t rc = BuildBlocks(res, blocks);
1152 delete res;
1153
1154 return rc;
1155 }
1156
1157 Int_t Build()
1158 {
1159 //get all dates from the database
1160 TSQLResult *res = Query("SELECT fDate FROM SequenceBuildStatus ORDER BY fDate DESC");
1161 if (!res)
1162 return 2;
1163
1164 //execute buildsequenceentries for all dates
1165 TSQLRow *row=0;
1166 while ((row=res->Next()))
1167 Build((*row)[0]);
1168
1169 delete res;
1170
1171 return 1;
1172 }
1173 ClassDef(SequenceBuild, 0)
1174};
1175
1176ClassImp(SequenceBuild);
1177
1178int buildsequenceentries(TString day, TString datapath, TString sequpath, Bool_t dummy=kTRUE)
1179{
1180 TEnv env("sql.rc");
1181
1182 SequenceBuild serv(env);
1183 if (!serv.IsConnected())
1184 {
1185 cout << "ERROR - Connection to database failed." << endl;
1186 return 0;
1187 }
1188
1189 serv.SetIsDummy(dummy);
1190 serv.SetPathRawData(datapath);
1191 serv.SetPathSequences(sequpath);
1192 return serv.Build(day);
1193}
1194
1195//
1196// Build Sequences for all Nights
1197//
1198int buildsequenceentries(TString datapath, TString sequpath, Bool_t dummy=kTRUE)
1199{
1200 TEnv env("sql.rc");
1201
1202 SequenceBuild serv(env);
1203 if (!serv.IsConnected())
1204 {
1205 cout << "ERROR - Connection to database failed." << endl;
1206 return 0;
1207 }
1208
1209 serv.SetIsDummy(dummy);
1210 serv.SetPathRawData(datapath);
1211 serv.SetPathSequences(sequpath);
1212 return serv.Build();
1213}
1214
1215int buildsequenceentries(Bool_t dummy=kTRUE)
1216{
1217 return buildsequenceentries("", "", dummy);
1218}
1219
1220int buildsequenceentries(TString day, Bool_t dummy=kTRUE)
1221{
1222 return buildsequenceentries(day, "", "", dummy);
1223}
Note: See TracBrowser for help on using the repository browser.