source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 8030

Last change on this file since 8030 was 8010, checked in by Daniela Dorner, 19 years ago
*** empty log message ***
File size: 37.0 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2006
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82
83#include <TSQLRow.h>
84#include <TSQLResult.h>
85
86#include <TEnv.h>
87#include <TMap.h>
88#include <TMath.h>
89#include <TExMap.h>
90#include <TArrayI.h>
91#include <TArrayD.h>
92#include <TPRegexp.h>
93#include <TSystem.h>
94#include <TObjString.h>
95#include <TObjArray.h>
96
97#include "MTime.h"
98#include "MDirIter.h"
99
100#include "MSQLMagic.h"
101
102using namespace std;
103
104
105class Rule : public TObject
106{
107private:
108 TPRegexp fRegexp;
109
110 UInt_t fMin;
111 UInt_t fMax;
112
113public:
114 Rule(const char *regexp, UInt_t min=0, UInt_t max=(UInt_t)-1) :
115 fRegexp(Form("^%s", regexp)), fMin(min), fMax(max)
116 {
117 }
118
119 Rule(TObjArray &arr) :
120 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
121 fMin(0), fMax((UInt_t)-1)
122 {
123 if (arr.GetEntries()>1)
124 fMin = atoi(arr[1]->GetName());
125 if (arr.GetEntries()>2)
126 fMax = atoi(arr[2]->GetName());
127
128 //cout << "Regexp: " << (arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):"") << " " << fMin << " " << fMax << endl;
129 }
130
131 Ssiz_t Match(const TString &str, Int_t idx, Int_t run=-1)
132 {
133 if (!IsValid(run))
134 return 0;
135
136 TString mods;
137 TArrayI pos;
138 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
139
140 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
141 }
142
143 Bool_t IsValid(UInt_t run) const { return run<0 || (run>=fMin && run<=fMax); }
144 ClassDef(Rule, 0)
145};
146ClassImp(Rule);
147
148class CheckMatch : public TObject
149{
150private:
151 TPRegexp fRunType1;
152 TPRegexp fRunType2;
153
154 TPRegexp fRegexp1;
155 TPRegexp fRegexp2;
156
157 UInt_t fMin;
158 UInt_t fMax;
159
160 void Init(const TObjArray &arr, Int_t idx=0)
161 {
162 const Int_t n = arr.GetEntries();
163
164 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
165
166 for (int i=0; i<n-idx; i++)
167 {
168 //cout << arr[i+idx]->GetName() << " ";
169
170 TString str(arr[i+idx]->GetName());
171 if (str=="*")
172 str = ".*";
173
174 switch (isminus && i>1 ? i+1 : i)
175 {
176 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
177 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
178 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
179 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
180 case 4: fMin = str.Atoi(); break;
181 case 5: fMax = str.Atoi(); break;
182 }
183 }
184 //cout << endl;
185 }
186
187public:
188 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
189
190 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
191 {
192 TObjArray *arr = txt.Tokenize(" ");
193 Init(*arr);
194 delete arr;
195 }
196
197 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
198 {
199 Init(arr, idx);
200 }
201
202 CheckMatch(const char *from, const char *to, Int_t min=0, Int_t max=-1)
203 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
204 {
205 }
206
207 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, UInt_t run=0)
208 {
209 if (run>0)
210 {
211 if (run<fMin || run>fMax)
212 return kFALSE;
213 }
214
215 const TString test("X-"); // FIXME:STUPID!
216
217 //if (test.Index(fRunType2)==1)
218 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
219
220 if (test.Index(fRunType1,0)==1)
221 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
222
223 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
224 }
225 ClassDef(CheckMatch,0)
226};
227ClassImp(CheckMatch);
228
229class CheckList : public TList
230{
231public:
232 CheckList() { SetOwner(); }
233 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Int_t run=-1) const
234 {
235 TIter Next(this);
236
237 CheckMatch *check = 0;
238
239 while ((check=(CheckMatch*)Next()))
240 {
241 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
242 if (rc)
243 return rc;
244 }
245
246 return kFALSE;
247 }
248 ClassDef(CheckList,0)
249};
250ClassImp(CheckList);
251
252class SequenceBuild : public MSQLMagic
253{
254private:
255 TString fPathRawData;
256 TString fPathSequences;
257
258 TMap fMap;
259 TList fListRegexp;
260
261 /*
262 TString GetKeyName(TString col, TString key)
263 {
264 col(TRegexp("^Elt")) = "";
265 return QueryNameOfKey(col, key, kFALSE);
266 }*/
267
268 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
269 {
270 // Get the name of the column from result table
271 const TString key = res.GetFieldName(i);
272
273 // Get the list with the allowed attributed from the map
274 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
275 if (!list)
276 return kFALSE;
277
278 // Check whether the current run (row[0]) with the current attribute
279 // (row[i]) and the current run-type (row[1]) matches the attribute
280 // of the last run (keys[i] with run-type row[i])
281 return list->Matches(keys[1], row[1], keys[i], row[i], atoi(row[0]));
282 }
283
284 Bool_t InsertSequence(Int_t from, Int_t to)
285 {
286 cout << " - Inserting Sequence into database." << endl;
287
288 // ========== Request number of events ==========
289
290 const TString runtime =
291 "SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
292 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
293 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ";
294
295 const TString where = Form("(fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
296 from, to);
297
298 TString query;
299 query = "SELECT SUM(fNumEvents), ";
300 query += runtime;
301 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
302 query += " MIN(fAzimuth), MAX(fAzimuth), ";
303 query += " MIN(fRunStart), MAX(fRunStop), ";
304 query += " ELT(MAX(FIELD(fLightConditionsKEY, 1, 2, 7, 5, 8)), 1, 2, 7, 5, 8) ";
305 query += " FROM RunData WHERE ";
306 query += where;
307 query += " AND fRunTypeKEY=2";
308
309 TSQLResult *res = Query(query);
310 if (!res)
311 return kFALSE;
312
313 TSQLRow *row = res->Next();
314 if (!row || res->GetFieldCount()!=9)
315 {
316 cout << "ERROR - Wrong result from query: " << query << endl;
317 return kFALSE;
318 }
319
320 const TString nevts = (*row)[0];
321 const TString secs = (*row)[1];
322 const TString zdmin = (*row)[2];
323 const TString zdmax = (*row)[3];
324 const TString azmin = (*row)[4];
325 const TString azmax = (*row)[5];
326 const TString start = (*row)[6];
327 const TString stop = (*row)[7];
328 const TString light = (*row)[8];
329
330 delete res;
331
332 // ========== Request data of sequence ==========
333 query = Form("SELECT fSourceKEY, fProjectKEY, "
334 " fL1TriggerTableKEY, fL2TriggerTableKEY,"
335 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
336 " fTriggerDelayTableKEY, fObservationModeKEY "
337 " FROM RunData WHERE fRunTypeKEY=2 AND %s"
338 " LIMIT 1", where.Data());
339
340 res = Query(query);
341 if (!res)
342 return kFALSE;
343
344 row = res->Next();
345 if (!row || res->GetFieldCount()!=8)
346 {
347 cout << "ERROR - No result from query: " << query << endl;
348 return kFALSE;
349 }
350
351 const TString set = Form("fSequenceFirst=%d ", from);
352
353 TString query1;
354 query1 += set;
355 query1 += Form(",fSequenceLast=%d,", to);
356 query1 += Form(" fSourceKEY=%s,", (*row)[0]);
357 query1 += Form(" fProjectKEY=%s,", (*row)[1]);
358 query1 += Form(" fNumEvents=%s,", nevts.Data());
359 query1 += Form(" fRunTime=%s,", secs.Data());
360 query1 += Form(" fRunStart=\"%s\",", start.Data());
361 query1 += Form(" fRunStop=\"%s\",", stop.Data());
362 query1 += Form(" fZenithDistanceMin=%s,", zdmin.Data());
363 query1 += Form(" fZenithDistanceMax=%s,", zdmax.Data());
364 query1 += Form(" fAzimuthMin=%s,", azmin.Data());
365 query1 += Form(" fAzimuthMax=%s,", azmax.Data());
366 query1 += Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
367 query1 += Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
368 query1 += Form(" fHvSettingsKEY=%s,", (*row)[4]);
369 query1 += Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
370 query1 += Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
371 query1 += Form(" fLightConditionsKEY=%s,", light.Data());
372 query1 += Form(" fObservationModeKEY=%s, ", (*row)[7]);
373 query1 += "fManuallyChangedKEY=1";
374
375 delete res;
376
377 const TString where2 = Form("(fRunTypeKEY BETWEEN 2 AND 4) AND %s",
378 where.Data());
379
380 if (!Insert("Sequences", query1))
381 {
382 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
383 return kFALSE;
384 }
385
386 if (!Update("RunData", set, where))
387 {
388 cout << "ERROR - Could not update RunData." << endl;
389 return kFALSE;
390 }
391
392 if (!Insert("SequenceProcessStatus", set))
393 {
394 cout << "ERROR - Could not insert Sequence into SequenceProcessStatus." << endl;
395 return kFALSE;
396 }
397
398 return kTRUE;
399 }
400
401 Bool_t DeleteSequence(Int_t sequ)
402 {
403 if (fPathRawData.IsNull() || fPathSequences.IsNull())
404 {
405 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
406 return kTRUE;
407 }
408
409 //queries to delete information from the database
410 const TString query(Form("fSequenceFirst=%d", sequ));
411
412 //commands to delete files from the disk
413 const TString fname(Form("%s/%04d/sequence%08d.txt", fPathSequences.Data(),sequ/10000, sequ));
414 const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
415 const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
416
417 if (!Delete("Calibration", query))
418 return 2;
419
420 if (!Delete("Star", query))
421 return 2;
422
423 if (!Delete("SequenceProcessStatus", query))
424 return 2;
425
426 if (!Delete("Sequences", query))
427 return 2;
428
429 if (!Update("RunData", "fSequenceFirst=0", query))
430 return 2;
431
432 if (IsDummy())
433 {
434 cout << " + unlink " << fname << endl;
435 cout << " + " << cmd1 << endl;
436 cout << " + " << cmd2 << endl;
437 return kTRUE;
438 }
439
440 gSystem->Unlink(fname);
441
442 gSystem->Exec(cmd1);
443 gSystem->Exec(cmd2);
444
445 return kTRUE;
446 }
447
448 Int_t CheckSequence(Int_t runstart, Int_t runstop)
449 {
450 const char *fmt1 = "SELECT fRunNumber FROM RunData WHERE";
451 const char *fmt2 = "AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) ORDER BY fRunNumber";
452
453 const TString query1 = Form("%s fSequenceFirst=%d %s", fmt1, runstart, fmt2);
454 const TString query2 = Form("%s fRunNumber BETWEEN %d AND %d %s", fmt1, runstart, runstop, fmt2);
455
456 TSQLResult *res1 = Query(query1);
457 if (!res1)
458 return 2;
459
460 TSQLResult *res2 = Query(query2);
461 if (!res2)
462 {
463 delete res1;
464 return 2;
465 }
466
467 while (1)
468 {
469 TSQLRow *row1 = res1->Next();
470 TSQLRow *row2 = res2->Next();
471
472 if (!row1 && !row2)
473 return kTRUE;
474
475 if (!row1 || !row2)
476 return kFALSE;
477
478 if (atoi((*row1)[0])!=atoi((*row2)[0]))
479 return kFALSE;
480 }
481
482 return kFALSE;
483 }
484
485 Int_t CreateSequence(Int_t runstart, Int_t runstop)
486 {
487 cout << " * Creating Sequence " << runstart << "-" << runstop << ":" << endl;
488
489 TString query=
490 Form("SELECT fSequenceFirst FROM RunData "
491 " WHERE fRunNumber BETWEEN %d AND %d AND "
492 " fSequenceFirst>0 AND "
493 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
494 " GROUP BY fSequenceFirst", runstart, runstop);
495
496 TSQLResult *res = Query(query);
497 if (!res)
498 return 2;
499
500 const Int_t cnt = res->GetRowCount();
501
502 Int_t rc = kTRUE;
503 if (cnt==1)
504 {
505 TSQLRow *row=res->Next();
506 const Int_t check = CheckSequence(runstart, runstop);
507 if (check==kTRUE)
508 {
509 cout << " - Identical sequence already existing." << endl;
510 delete res;
511 return kTRUE;
512 }
513 if (check==2)
514 rc=2;
515 else
516 {
517 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
518 if (DeleteSequence(atoi((*row)[0]))==2)
519 rc = 2;
520 }
521 }
522 else
523 {
524 TSQLRow *row=0;
525 while ((row=res->Next()))
526 {
527 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
528 if (DeleteSequence(atoi((*row)[0]))==2)
529 rc = 2;
530 }
531 }
532
533 delete res;
534
535 if (rc==2)
536 return 2;
537
538 if (!InsertSequence(runstart, runstop))
539 return 2;
540
541 return kTRUE;
542 }
543
544 Bool_t ReadResources(const char *fname)
545 {
546 TPRegexp regexp("^\\[.*\\]$");
547
548 ifstream fin(fname);
549 if (!fin)
550 {
551 cout << "Cannot open file " << fname << ": ";
552 cout << strerror(errno) << endl;
553 return kFALSE;
554 }
555
556 Int_t section = 0;
557
558 TString key;
559 while (1)
560 {
561 TString txt;
562 txt.ReadLine(fin);
563 if (!fin)
564 break;
565
566 txt = txt.Strip(TString::kBoth);
567
568 if (txt[0]=='#' || txt.IsNull())
569 continue;
570
571 if (txt[0]=='[' && section!=2)
572 {
573 //cout << txt << endl;
574 section = 0;
575 if (txt(regexp)=="[Transition]")
576 section = 1;
577 if (txt(regexp)=="[Regexp]")
578 section = 2;
579 continue;
580 }
581
582 TObjArray *arr = txt.Tokenize(" ");
583
584 if (arr->GetEntries()>0)
585 switch (section)
586 {
587 case 1:
588 {
589 TString key = arr->At(0)->GetName();
590 key.Prepend("f");
591 key.Append("KEY");
592
593 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
594 if (!list)
595 {
596 //cout << key << endl;
597 list = new CheckList;
598 fMap.Add(new TObjString(key), list);
599 }
600
601 if (arr->GetEntries()>1)
602 {
603 //cout << key << " ";
604 list->Add(new CheckMatch(*arr, 1));
605 }
606 }
607 break;
608 case 2:
609 fListRegexp.Add(new Rule(*arr));
610 break;
611 }
612
613 delete arr;
614 }
615
616 return kTRUE;
617 }
618
619 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
620 {
621 TObjArray names; // array with old names (including regexp)
622
623 // Add to array and expand the array if necessary
624 TSQLRow *row=0;
625 while ((row=res->Next()))
626 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
627
628 // Now a LUT is build which converts the keys for
629 // the names including the regexp into keys for
630 // the names excluding the regexp
631 TString elt(Form("ELT(RunData.f%sKEY+1", col));
632
633 // loop over all entries in the list
634 const Int_t n = names.GetSize();
635 for (int i=0; i<n; i++)
636 {
637 // For all entries which are not in the list
638 // write an undefined value into the LUT
639 TObject *o = names.UncheckedAt(i);
640 if (!o)
641 {
642 elt += ",0";
643 continue;
644 }
645
646 // Remove the regexp from the string which includes it
647 TString name = o->GetName();
648
649 TIter NextR(&regexp);
650 TObject *reg=0;
651 while ((reg=NextR()))
652 {
653 TPRegexp reg(reg->GetName());
654 const Ssiz_t pos = name.Index(reg, 0);
655 if (pos>0)
656 {
657 name.Remove(pos);
658 name += "-W";
659 break;
660 }
661 }
662
663 // Check if such a Key exists, if not insert it
664 const Int_t key = QueryKeyOfName(col, name);
665
666 // RESOLVE return code!!!
667 //if (key<0)
668 // return "";
669
670 // add index to the LUT
671 elt += Form(",%d", key);
672 }
673
674 // close LUT expression
675 // elt += ") AS Elt";
676 // elt += col;
677
678 elt += ") AS f";
679 elt += col;
680 elt += "KEY";
681
682 // return result
683 return elt;
684 }
685
686 TString GetELT(const char *col, TSQLResult *res, TString regexp)
687 {
688 TList list;
689 list.SetOwner();
690 list.Add(new TObjString(regexp));
691 return GetELT(col, res, list);
692 }
693
694 Bool_t HasAtLeastOne(TString src, TString chk) const
695 {
696 src.ToLower();
697 chk.ToLower();
698
699 for (int i=0; i<chk.Length(); i++)
700 if (src.First(chk[i])<0)
701 return kFALSE;
702
703 return kTRUE;
704 }
705
706 TString PrepareString(TSQLResult &res, TArrayI &runs)
707 {
708 // Number of result rows
709 const Int_t rows = res.GetRowCount();
710
711 runs.Set(rows); // initialize size of array for run numbers
712
713 TArrayD start(rows); // instantiate array for start times
714 TArrayD stop(rows); // instantiate array for stop times
715
716 TString str; // result string
717
718 Int_t idx=0;
719 TSQLRow *row=0;
720 while ((row=res.Next()))
721 {
722 runs[idx] = atoi((*row)[0]); // run number
723
724 const TString tstart = ((*row)[2]); // start time
725 const TString tstop = ((*row)[3]); // stop time
726
727 start[idx] = MTime(tstart).GetMjd(); // convert to double
728 stop[idx] = MTime(tstop).GetMjd(); // convert to double
729
730 // This is a workaround for broken start-times
731 if (tstart=="0000-00-00 00:00:00")
732 start[idx] = stop[idx];
733
734 // Add a run-type character for this run to the string
735 str += RunType((*row)[1]);
736
737 // Increase index
738 idx++;
739 }
740
741 // Now the P- and D- runs are classified by the time-distance
742 // to the next or previous C-Run
743 Double_t lastc = -1;
744 Double_t nextc = -1;
745 for (int i=0; i<str.Length(); i++)
746 {
747 if (str[i]=='C')
748 {
749 // Remember stop time of C-Run as time of last C-run
750 lastc = stop[i];
751
752 // Calculate start time of next C-Run
753 const TString residual = str(i+1, str.Length());
754 const Int_t pos = residual.First('C');
755
756 // No C-Run found anymore. Finished...
757 if (pos<0)
758 break;
759
760 // Remember start time of C-Run as time of next C-run
761 nextc = start[i+1+pos];
762 continue;
763 }
764
765 // Check whether the identifying character should
766 // be converted to lower case
767 if (start[i]-lastc>nextc-stop[i])
768 str[i] = tolower(str[i]);
769 }
770 //cout << str << endl;
771 return str;
772 }
773
774 void PrintResidual(Int_t runstart, Int_t runstop, TString residual, const char *descr)
775 {
776 residual.ToLower();
777
778 // Count number of unsequences "characters"
779 const Int_t nump = residual.CountChar('p');
780 const Int_t numd = residual.CountChar('d');
781 const Int_t numc = residual.CountChar('c');
782
783 // Print some information to the output steram
784 if (nump+numc+numd==0)
785 return;
786
787 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
788 if (numd>0)
789 cout << " (DATA!)";
790
791 if (nump==0 || numc==0 || numd==0)
792 cout << " Missing";
793 if (numd==0)
794 cout << " D";
795 if (nump==0)
796 cout << " P";
797 if (numc==0)
798 cout << " C";
799
800 cout << endl;
801 }
802
803 Int_t SplitBlock(Int_t runstart, Int_t runstop)
804 {
805 // Request data necessary to split block into sequences
806 const TString query=
807 Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
808 " FROM RunData "
809 " WHERE fRunNumber BETWEEN %d AND %d AND "
810 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
811 " ORDER BY fRunNumber", runstart, runstop);
812
813 // Send query
814 TSQLResult *res = Query(query);
815 if (!res)
816 return 2;
817
818 // Get String containing the sequence of P-,C- and D-Runs
819 // and an array with the corresponding run-numbers
820 TArrayI runs;
821 const TString str = PrepareString(*res, runs);
822
823 delete res;
824
825 // Check if the prepared string at least contains one run of each type
826 if (!HasAtLeastOne(str, "PCD"))
827 {
828 PrintResidual(runstart, runstop, str, "Block");
829 return kTRUE;
830 }
831
832 // Start the sequence building
833 // ---------------------------
834
835 Int_t pos = 0; // Position in the string
836 TExMap map; // Map for resulting sequences
837 TString residual; // Unsequences "characters"
838
839 // Step through the string one character by one
840 cout << " ";
841 while (pos<str.Length())
842 {
843 Bool_t found = kFALSE;
844
845 // Loop over the predefined regular expressions
846 TIter Next(&fListRegexp);
847 Rule *obj = 0;
848 while ((obj=(Rule*)Next()))
849 {
850 // Check if regular expressions matchs
851 const Ssiz_t len = obj->Match(str, pos, runs[pos]);
852 if (len>0)
853 {
854 // In case of match get the substring which
855 // defines the sequence and print it
856 TString sub = str(pos, len);
857 sub.ToUpper();
858 cout << runs[pos]<<":"<< sub << "|";
859
860 // Add the first and last run of the sequence to the map
861 map.Add(runs[pos], runs[pos+len-1]);
862
863 // A sequence was found...
864 found = kTRUE;
865
866 // step forward in the string by the length of the sequence
867 pos += len;
868 break;
869 }
870 }
871
872 // If a sequence was found go on...
873 if (found)
874 continue;
875
876 // print unsequenced characters
877 cout << (char)tolower(str[pos]);
878
879 // Count the number of "characters" not sequenced
880 residual += str[pos];
881
882 // step one character forward
883 pos++;
884 }
885 cout << endl;
886
887 PrintResidual(runstart, runstop, residual, "Runs ");
888
889 // Create all sequences which were previously found
890 Long_t first, last;
891 TExMapIter iter(&map);
892 while (iter.Next(first, last))
893 if (CreateSequence(first, last)==2)
894 return 2;
895
896 return kTRUE;
897 }
898
899 Char_t RunType(Int_t n) const
900 {
901 switch (n)
902 {
903 case 2: return 'D';
904 case 3: return 'P';
905 case 4: return 'C';
906 }
907 return '-';
908 }
909 Char_t RunType(const char *str) const
910 {
911 return RunType(atoi(str));
912 }
913
914 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks)
915 {
916 // col key content
917 // -----------------------------
918 // 0 - runnumber
919 // 1 0 RunTypeKEY
920 // 2 1 source
921 // 3 2 project
922 // . .
923 // . .
924 // . . from transition.txt
925 //
926
927 //build blocks of runs, which have the same values
928 //for each block the first and the last run are stored in a TExMap
929 //the values are checked with the help of an array of TStrings
930 Long_t runstart = -1;
931 Long_t runstop = -1;
932
933 const UInt_t ncols = res->GetFieldCount();
934
935 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
936
937 // Loop over runs
938 TSQLRow *row=0;
939 while ((row=res->Next()))
940 {
941 // This is the runnumber of the first run in the new block
942 if (runstart<0)
943 runstart=atoi((*row)[0]);
944
945 // Check which transitions might be allowed for this run and
946 // which are not. Check also which attributes should be ignored
947 // for this run. The information about it is stored in the map.
948 Int_t rc[ncols];
949 for (UInt_t i=2; i<ncols; i++)
950 rc[i] = CheckTransition(*res, keys, *row, i);
951
952 for (UInt_t i=2; i<ncols; i++)
953 {
954 switch (rc[i])
955 {
956 case kTRUE: // Transition allowed, switch to new attribute
957 keys[i] = (*row)[i];
958 continue;
959
960 case -1: // Ignore attribute, continue with next one
961 //if (keys[i] == (*row)[i])
962 // break; // Only ignore it if it is really different
963 continue;
964
965 case kFALSE: // No decision (nothing matching found in map)
966 break; // go on checking
967
968 }
969
970 // If past-attribute is not yet initialized, do it now.
971 if (keys[i].IsNull())
972 keys[i] = (*row)[i];
973
974 // Check whether key has changed for this run
975 // (this condition is never true for the first run)
976 // This condition in only reached if keys[i] was initialized
977 if (keys[i] == (*row)[i])
978 continue;
979
980 // Found one block with unique keys, fill values into TExMap
981 // (except if this is the first run processed)
982 cout << endl;
983 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
984 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
985 if (SplitBlock(runstart, runstop)==2)
986 return 2;
987 cout << " - Transition from " << RunType(keys[1]) << ":";
988 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
989 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
990 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
991
992 // This is already the first run of the new block
993 runstart=atoi((*row)[0]);
994
995 // These are the keys corresponding to the first run
996 // in the new block. All following runs should have
997 // identical keys. Do not set the attribute if for
998 // this attribute the transition check evaluated
999 // to "ignore".
1000 for (UInt_t j=2; j<ncols; j++)
1001 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1002
1003 break;
1004 }
1005
1006 keys[1] = (*row)[1]; // Remember run type for next run
1007
1008 // This is the new runnumber of the last run in this block
1009 runstop=atoi((*row)[0]);
1010 }
1011
1012 // If runstart==-1 a possible block would contain a single run...
1013 if (runstart>0 && runstart!=runstop)
1014 {
1015 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1016 //fill values into TExMap (last value)
1017 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1018 if (SplitBlock(runstart, runstop)==2)
1019 return 2;
1020 }
1021 cout << "Done." << endl << endl;
1022
1023 return kTRUE;
1024 }
1025
1026public:
1027 SequenceBuild(TEnv &env) : MSQLMagic(env)
1028 {
1029 cout << "buildsequences" << endl;
1030 cout << "--------------" << endl;
1031 cout << endl;
1032 cout << "Connected to " << GetName() << endl;
1033
1034 fListRegexp.SetOwner();
1035
1036 // FIXME: THIS IS NOT YET HANDLED
1037 if (ReadResources("resources/sequences.rc"))
1038 return;
1039 }
1040 SequenceBuild()
1041 {
1042 fMap.DeleteAll();
1043 }
1044 /*
1045 SequenceBuild(SequenceBuild &sb) : MSQLServer(sb)
1046 {
1047 fPathRawData = sb.fPathRawData;
1048 fPathSequences = sb.fPathSequences;
1049 }*/
1050 void SetPathRawData(const char *path) { fPathRawData=path; }
1051 void SetPathSequences(const char *path) { fPathSequences=path; }
1052
1053 int Build(TString day)
1054 {
1055 cout << endl;
1056 cout << "Night of sunrise at " << day << ":" << endl;
1057
1058 day += " 13:00:00";
1059 const TString cond =
1060 Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\") "
1061 "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 "
1062 "ORDER BY fRunNumber",
1063 day.Data(), day.Data());
1064
1065
1066
1067 //query all sources observed in this night
1068 TString querys(Form("SELECT RunData.fSourceKEY, fSourceName FROM RunData "
1069 "LEFT JOIN Source ON RunData.fSourceKEY=Source.fSourceKEY "
1070 "WHERE %s", cond.Data()));
1071 TSQLResult *resx = Query(querys);
1072 if (!resx)
1073 return 2;
1074 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
1075 delete resx;
1076
1077 //query all project names observed in this night
1078 TString queryp(Form("SELECT RunData.fProjectKEY, fProjectName FROM RunData "
1079 "LEFT JOIN Project ON RunData.fProjectKEY=Project.fProjectKEY "
1080 "WHERE %s", cond.Data()));
1081 resx = Query(queryp);
1082 if (!resx)
1083 return 2;
1084
1085 TList regexp;
1086 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
1087 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
1088 TString eltp2 = GetELT("Project", resx, regexp);
1089 delete resx;
1090 regexp.Delete();
1091
1092 // Setup query to get all values from the database,
1093 // that are relevant for building sequences
1094 TString query("SELECT fRunNumber, fRunTypeKEY, ");
1095 query += elts;
1096 query += ", ";
1097 query += eltp2;
1098
1099 // Now add all entries from the transition table to the query
1100 TIter NextPair(&fMap);
1101 TObject *mapkey = 0;
1102 while ((mapkey=(TPair*)NextPair()))
1103 if (!query.Contains(mapkey->GetName()))
1104 {
1105 query += ", ";
1106 query += mapkey->GetName();
1107 }
1108 query += Form(" FROM RunData WHERE %s", cond.Data());
1109
1110 TSQLResult *res = Query(query);
1111 if (!res)
1112 return 2;
1113
1114 TExMap blocks;
1115 const Int_t rc = BuildBlocks(res, blocks);
1116 delete res;
1117
1118 return rc;
1119 }
1120
1121 Int_t Build()
1122 {
1123 //get all dates from the database
1124 TSQLResult *res = Query("SELECT fDate FROM SequenceBuildStatus ORDER BY fDate DESC");
1125 if (!res)
1126 return 2;
1127
1128 //execute buildsequenceentries for all dates
1129 TSQLRow *row=0;
1130 while ((row=res->Next()))
1131 Build((*row)[0]);
1132
1133 delete res;
1134
1135 return 1;
1136 }
1137 ClassDef(SequenceBuild, 0)
1138};
1139
1140ClassImp(SequenceBuild);
1141
1142int buildsequenceentries(TString day, TString datapath, TString sequpath, Bool_t dummy=kTRUE)
1143{
1144 TEnv env("sql.rc");
1145
1146 SequenceBuild serv(env);
1147 if (!serv.IsConnected())
1148 {
1149 cout << "ERROR - Connection to database failed." << endl;
1150 return 0;
1151 }
1152
1153 serv.SetIsDummy(dummy);
1154 serv.SetPathRawData(datapath);
1155 serv.SetPathSequences(sequpath);
1156 return serv.Build(day);
1157}
1158
1159//
1160// Build Sequences for all Nights
1161//
1162int buildsequenceentries(TString datapath, TString sequpath, Bool_t dummy=kTRUE)
1163{
1164 TEnv env("sql.rc");
1165
1166 SequenceBuild serv(env);
1167 if (!serv.IsConnected())
1168 {
1169 cout << "ERROR - Connection to database failed." << endl;
1170 return 0;
1171 }
1172
1173 serv.SetIsDummy(dummy);
1174 serv.SetPathRawData(datapath);
1175 serv.SetPathSequences(sequpath);
1176 return serv.Build();
1177}
1178
1179int buildsequenceentries(Bool_t dummy=kTRUE)
1180{
1181 return buildsequenceentries("", "", dummy);
1182}
1183
1184int buildsequenceentries(TString day, Bool_t dummy=kTRUE)
1185{
1186 return buildsequenceentries(day, "", "", dummy);
1187}
Note: See TracBrowser for help on using the repository browser.