source: trunk/MagicSoft/Mars/datacenter/macros/buildsequenceentries.C@ 8198

Last change on this file since 8198 was 8191, checked in by Daniela Dorner, 19 years ago
*** empty log message ***
File size: 37.6 KB
Line 
1/* ======================================================================== *\
2!
3! *
4! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5! * Software. It is distributed to you in the hope that it can be a useful
6! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7! * It is distributed WITHOUT ANY WARRANTY.
8! *
9! * Permission to use, copy, modify and distribute this software and its
10! * documentation for any purpose is hereby granted without fee,
11! * provided that the above copyright notice appear in all copies and
12! * that both that copyright notice and this permission notice appear
13! * in supporting documentation. It is provided "as is" without express
14! * or implied warranty.
15! *
16!
17!
18! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
19! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
20!
21! Copyright: MAGIC Software Development, 2000-2006
22!
23!
24\* ======================================================================== */
25
26/////////////////////////////////////////////////////////////////////////////
27//
28// buildsequenceentries.C
29// ======================
30//
31// to group the runs of one night into sequences, this marco:
32// - reads the runinformation of one night from the database
33// - group the runs into sets of following runs with the same conditions
34// - groups the runs in this sets to sequences such that each run belongs
35// to the nearest (in time) calibration run
36// - check if the runs with the same runtype have the same calibration script
37// and the same trigger tables
38// if sequence is okay:
39// - check if in the range of the runnumbers of this sequence other sequences
40// exist in the database
41// if there are no sequences, insert the new sequence, else:
42// - delete overlaping sequences
43// if there's only one sequence in the same runnumber range:
44// - check if the new and the old sequence are identical
45// if they are identical, do nothing, if not, delete the old sequence and
46// insert the new one
47//
48// remark: deleting sequences includes the following steps:
49// - delete entries from the tables Sequences, SequenceProcessStatus,
50// Calibration and Star
51// - updating the sequence number (fSequenceFirst) in the table RunData
52// - remove the Sequence File, the calibrated data and the image files from
53// the disk
54//
55// the macro can be executed either for all nights or for one single night
56// .x buildsequenceentries.C+( "datapath", "sequpath", Bool_t dummy=kTRUE)
57// .x buildsequenceentries.C+( "night", "datapath", "sequpath")
58//
59// the Bool_t dummy:
60// kTRUE: dummy-mode, i.e. nothing is inserted into the database, but the
61// commands, that would be executed are returned
62// kFALSE: the information is inserted into the database and the files of
63// removed sequences is deleted
64// be careful with this option - for tests use always kTRUE
65//
66// TString datapath, TString sequpath:
67// datapath: path, where the processed data is stored in the datacenter
68// sequpath: path, where the sequence files are stored in the datacenter
69// the datapath (standard: /magic/data/) and the sequencepath (standard:
70// /magic/sequences) have to be given, that the sequence file, the
71// calibrated data and the star files can be removed, when an old sequence
72// has to be removed from the database
73//
74// If nothing failes 1 is returned. In the case of an error 2 and if
75// there's no connection to the database 0 is returned.
76// This is needed for the scripts that execute the macro.
77//
78/////////////////////////////////////////////////////////////////////////////
79#include <iostream>
80#include <iomanip>
81#include <fstream>
82
83#include <TSQLRow.h>
84#include <TSQLResult.h>
85
86#include <TEnv.h>
87#include <TMap.h>
88#include <TMath.h>
89#include <TExMap.h>
90#include <TArrayI.h>
91#include <TArrayD.h>
92#include <TPRegexp.h>
93#include <TSystem.h>
94#include <TObjString.h>
95#include <TObjArray.h>
96
97#include "MTime.h"
98#include "MDirIter.h"
99
100#include "MSQLMagic.h"
101
102using namespace std;
103
104
105class Rule : public TObject
106{
107private:
108 TPRegexp fRegexp;
109
110 UInt_t fMin;
111 UInt_t fMax;
112
113public:
114 Rule(const char *regexp, UInt_t min=0, UInt_t max=(UInt_t)-1) :
115 fRegexp(Form("^%s", regexp)), fMin(min), fMax(max)
116 {
117 }
118
119 Rule(TObjArray &arr) :
120 fRegexp(arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):""),
121 fMin(0), fMax((UInt_t)-1)
122 {
123 if (arr.GetEntries()>1)
124 fMin = atoi(arr[1]->GetName());
125 if (arr.GetEntries()>2)
126 fMax = atoi(arr[2]->GetName());
127
128 //cout << "Regexp: " << (arr.GetEntries()>0?Form("^%s", arr[0]->GetName()):"") << " " << fMin << " " << fMax << endl;
129 }
130
131 Ssiz_t Match(const TString &str, Int_t idx, Int_t run=-1)
132 {
133 if (!IsValid(run))
134 return 0;
135
136 TString mods;
137 TArrayI pos;
138 fRegexp.Match(str.Data()+idx, mods, 0, str.Length()*10, &pos);
139
140 return pos.GetSize()<2 ? 0 : pos[1]-pos[0];
141 }
142
143 Bool_t IsValid(UInt_t run) const { return run<0 || (run>=fMin && run<=fMax); }
144 ClassDef(Rule, 0)
145};
146ClassImp(Rule);
147
148class CheckMatch : public TObject
149{
150private:
151 TPRegexp fRunType1;
152 TPRegexp fRunType2;
153
154 TPRegexp fRegexp1;
155 TPRegexp fRegexp2;
156
157 UInt_t fMin;
158 UInt_t fMax;
159
160 void Init(const TObjArray &arr, Int_t idx=0)
161 {
162 const Int_t n = arr.GetEntries();
163
164 const Bool_t isminus = n>idx && TString(arr[idx]->GetName())=="-";
165
166 for (int i=0; i<n-idx; i++)
167 {
168 //cout << arr[i+idx]->GetName() << " ";
169
170 TString str(arr[i+idx]->GetName());
171 if (str=="*")
172 str = ".*";
173
174 switch (isminus && i>1 ? i+1 : i)
175 {
176 case 0: fRunType1 = TPRegexp(Form(isminus?"-":"^%s$", str.Data())); break;
177 case 1: fRunType2 = TPRegexp(Form("^%s$", str.Data())); break;
178 case 2: fRegexp1 = TPRegexp(Form("^%s$", str.Data())); break;
179 case 3: fRegexp2 = TPRegexp(Form("^%s$", str.Data())); break;
180 case 4: fMin = str.Atoi(); break;
181 case 5: fMax = str.Atoi(); break;
182 }
183 }
184 //cout << endl;
185 }
186
187public:
188 CheckMatch() : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2("") {}
189
190 CheckMatch(const TString &txt) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
191 {
192 TObjArray *arr = txt.Tokenize(" ");
193 Init(*arr);
194 delete arr;
195 }
196
197 CheckMatch(TObjArray &arr, Int_t idx=0) : fRunType1(""), fRunType2(""), fRegexp1(""), fRegexp2(""), fMin(0), fMax((UInt_t)-1)
198 {
199 Init(arr, idx);
200 }
201
202 CheckMatch(const char *from, const char *to, Int_t min=0, Int_t max=-1)
203 : fRunType1(".*"), fRunType2(".*"), fRegexp1(Form("^%s$", from)), fRegexp2(Form("^%s$", to)), fMin(min), fMax(max)
204 {
205 }
206
207 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, UInt_t run=0)
208 {
209 if (run>0)
210 {
211 if (run<fMin || run>fMax)
212 return kFALSE;
213 }
214
215 const TString test("X-"); // FIXME:STUPID!
216
217 //if (test.Index(fRunType2)==1)
218 // return -(!rt1(fRunType1).IsNull() && !lc1(fRegexp1).IsNull());
219
220 if (test.Index(fRunType1,0)==1)
221 return -(!rt2(fRunType2).IsNull() && !lc2(fRegexp2).IsNull());
222
223 return !rt1(fRunType1).IsNull() && !rt2(fRunType2).IsNull() && !lc1(fRegexp1).IsNull() && !lc2(fRegexp2).IsNull();
224 }
225 ClassDef(CheckMatch,0)
226};
227ClassImp(CheckMatch);
228
229class CheckList : public TList
230{
231public:
232 CheckList() { SetOwner(); }
233 Int_t Matches(const TString &rt1, const TString &rt2, const TString &lc1, const TString &lc2, Int_t run=-1) const
234 {
235 TIter Next(this);
236
237 CheckMatch *check = 0;
238
239 while ((check=(CheckMatch*)Next()))
240 {
241 const Int_t rc = check->Matches(rt1, rt2, lc1, lc2, run);
242 if (rc)
243 return rc;
244 }
245
246 return kFALSE;
247 }
248 ClassDef(CheckList,0)
249};
250ClassImp(CheckList);
251
252class SequenceBuild : public MSQLMagic
253{
254private:
255 TString fPathRawData;
256 TString fPathSequences;
257
258 TMap fMap;
259 TList fListRegexp;
260
261 Int_t CheckTransition(TSQLResult &res, const TString *keys, TSQLRow &row, Int_t i)
262 {
263 // Get the name of the column from result table
264 const TString key = res.GetFieldName(i);
265
266 // Get the list with the allowed attributed from the map
267 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
268 if (!list)
269 return kFALSE;
270
271 // Check whether the current run (row[0]) with the current attribute
272 // (row[i]) and the current run-type (row[1]) matches the attribute
273 // of the last run (keys[i] with run-type row[i])
274 return list->Matches(keys[1], row[1], keys[i], row[i], atoi(row[0]));
275 }
276
277 Bool_t InsertSequence(Int_t from, Int_t to)
278 {
279 cout << " - Inserting Sequence into database." << endl;
280
281 // ========== Request number of events ==========
282
283 const TString runtime =
284 "SUM(if(TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)<0,"
285 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart)+24*60*60,"
286 " TIME_TO_SEC(fRunStop)-TIME_TO_SEC(fRunStart))), ";
287
288 const TString where = Form("(fRunNumber BETWEEN %d AND %d) AND fExcludedFDAKEY=1",
289 from, to);
290
291 TString query;
292 query = "SELECT SUM(fNumEvents), ";
293 query += runtime;
294 query += " MIN(fZenithDistance), MAX(fZenithDistance), ";
295 query += " MIN(fAzimuth), MAX(fAzimuth), ";
296 query += " MIN(if(fRunStart=0,NULL,fRunStart)), ";
297 query += " MAX(if(fRunStop=0,NULL,fRunStop)), ";
298 query += " ELT(MAX(FIELD(fLightConditionsKEY, 1, 2, 7, 5, 8)), 1, 2, 7, 5, 8) ";
299 query += " FROM RunData WHERE ";
300 query += where;
301 query += " AND fRunTypeKEY=2";
302
303 TSQLResult *res = Query(query);
304 if (!res)
305 return kFALSE;
306
307 TSQLRow *row = res->Next();
308 if (!row || res->GetFieldCount()!=9)
309 {
310 cout << "ERROR - Wrong result from query: " << query << endl;
311 return kFALSE;
312 }
313
314 const TString nevts = (*row)[0];
315 const TString secs = (*row)[1];
316 const TString zdmin = (*row)[2];
317 const TString zdmax = (*row)[3];
318 const TString azmin = (*row)[4];
319 const TString azmax = (*row)[5];
320 const TString start = (*row)[6];
321 const TString stop = (*row)[7];
322 const TString light = (*row)[8];
323
324 delete res;
325
326 const TString elts = GetELTSource(where);
327 if (elts.IsNull())
328 return kFALSE;
329
330 const TString eltp = GetELTProject(where);
331 if (eltp.IsNull())
332 return kFALSE;
333
334 // ========== Request data of sequence ==========
335 query = "SELECT ";
336 query += elts;
337 query += ", ";
338 query += eltp;
339 query += ", fL1TriggerTableKEY, fL2TriggerTableKEY,"
340 " fHvSettingsKEY, fDiscriminatorThresholdTableKEY,"
341 " fTriggerDelayTableKEY, fObservationModeKEY "
342 " FROM RunData WHERE fRunTypeKEY=2 AND ";
343 query += where;
344 query += " LIMIT 1";
345
346 res = Query(query);
347 if (!res)
348 return kFALSE;
349
350 row = res->Next();
351 if (!row || res->GetFieldCount()!=8)
352 {
353 cout << "ERROR - No result from query: " << query << endl;
354 return kFALSE;
355 }
356
357 const TString set = Form("fSequenceFirst=%d ", from);
358
359 TString query1;
360 query1 += set;
361 query1 += Form(",fSequenceLast=%d,", to);
362 query1 += Form(" fSourceKEY=%s,", (*row)[0]);
363 query1 += Form(" fProjectKEY=%s,", (*row)[1]);
364 query1 += Form(" fNumEvents=%s,", nevts.Data());
365 query1 += Form(" fRunTime=%s,", secs.Data());
366 query1 += Form(" fRunStart=\"%s\",", start.Data());
367 query1 += Form(" fRunStop=\"%s\",", stop.Data());
368 query1 += Form(" fZenithDistanceMin=%s,", zdmin.Data());
369 query1 += Form(" fZenithDistanceMax=%s,", zdmax.Data());
370 query1 += Form(" fAzimuthMin=%s,", azmin.Data());
371 query1 += Form(" fAzimuthMax=%s,", azmax.Data());
372 query1 += Form(" fL1TriggerTableKEY=%s,", (*row)[2]);
373 query1 += Form(" fL2TriggerTableKEY=%s,", (*row)[3]);
374 query1 += Form(" fHvSettingsKEY=%s,", (*row)[4]);
375 query1 += Form(" fDiscriminatorThresholdTableKEY=%s,", (*row)[5]);
376 query1 += Form(" fTriggerDelayTableKEY=%s,", (*row)[6]);
377 query1 += Form(" fLightConditionsKEY=%s,", light.Data());
378 query1 += Form(" fObservationModeKEY=%s, ", (*row)[7]);
379 query1 += "fManuallyChangedKEY=1";
380
381 delete res;
382
383 const TString where2 = Form("(fRunTypeKEY BETWEEN 2 AND 4) AND %s",
384 where.Data());
385
386 if (!Insert("Sequences", query1))
387 {
388 cout << "ERROR - Could not insert Sequence into Sequences." << endl;
389 return kFALSE;
390 }
391
392 if (!Update("RunData", set, where))
393 {
394 cout << "ERROR - Could not update RunData." << endl;
395 return kFALSE;
396 }
397
398 if (!Insert("SequenceProcessStatus", set))
399 {
400 cout << "ERROR - Could not insert Sequence into SequenceProcessStatus." << endl;
401 return kFALSE;
402 }
403
404 return kTRUE;
405 }
406
407 Bool_t DeleteSequence(Int_t sequ)
408 {
409 if (fPathRawData.IsNull() || fPathSequences.IsNull())
410 {
411 cout << " + Deletion " << sequ << " skipped due to missing path." << endl;
412 return kTRUE;
413 }
414
415 //queries to delete information from the database
416 const TString query(Form("fSequenceFirst=%d", sequ));
417
418 //commands to delete files from the disk
419 const TString fname(Form("%s/%04d/sequence%08d.txt", fPathSequences.Data(),sequ/10000, sequ));
420 const TString cmd1(Form("rm -rf %s/callisto/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
421 const TString cmd2(Form("rm -rf %s/star/%04d/%08d/", fPathRawData.Data(), sequ/10000, sequ));
422
423 if (!Delete("Calibration", query))
424 return 2;
425
426 if (!Delete("Star", query))
427 return 2;
428
429 if (!Delete("SequenceProcessStatus", query))
430 return 2;
431
432 if (!Delete("Sequences", query))
433 return 2;
434
435 if (!Update("RunData", "fSequenceFirst=0", query))
436 return 2;
437
438 if (IsDummy())
439 {
440 cout << " + unlink " << fname << endl;
441 cout << " + " << cmd1 << endl;
442 cout << " + " << cmd2 << endl;
443 return kTRUE;
444 }
445
446 gSystem->Unlink(fname);
447
448 gSystem->Exec(cmd1);
449 gSystem->Exec(cmd2);
450
451 return kTRUE;
452 }
453
454 Int_t CheckSequence(Int_t runstart, Int_t runstop)
455 {
456 const char *fmt1 = "SELECT fRunNumber FROM RunData WHERE";
457 const char *fmt2 = "AND fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4) ORDER BY fRunNumber";
458
459 const TString query1 = Form("%s fSequenceFirst=%d %s", fmt1, runstart, fmt2);
460 const TString query2 = Form("%s fRunNumber BETWEEN %d AND %d %s", fmt1, runstart, runstop, fmt2);
461
462 TSQLResult *res1 = Query(query1);
463 if (!res1)
464 return 2;
465
466 TSQLResult *res2 = Query(query2);
467 if (!res2)
468 {
469 delete res1;
470 return 2;
471 }
472
473 while (1)
474 {
475 TSQLRow *row1 = res1->Next();
476 TSQLRow *row2 = res2->Next();
477
478 if (!row1 && !row2)
479 return kTRUE;
480
481 if (!row1 || !row2)
482 return kFALSE;
483
484 if (atoi((*row1)[0])!=atoi((*row2)[0]))
485 return kFALSE;
486 }
487
488 return kFALSE;
489 }
490
491 Int_t CreateSequence(Int_t runstart, Int_t runstop)
492 {
493 cout << " * Creating Sequence " << runstart << "-" << runstop << ":" << endl;
494
495 TString query=
496 Form("SELECT fSequenceFirst FROM RunData "
497 " WHERE fRunNumber BETWEEN %d AND %d AND "
498 " fSequenceFirst>0 AND "
499 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
500 " GROUP BY fSequenceFirst", runstart, runstop);
501
502 TSQLResult *res = Query(query);
503 if (!res)
504 return 2;
505
506 const Int_t cnt = res->GetRowCount();
507
508 Int_t rc = kTRUE;
509 if (cnt==1)
510 {
511 TSQLRow *row=res->Next();
512 const Int_t check = CheckSequence(runstart, runstop);
513 if (check==kTRUE)
514 {
515 cout << " - Identical sequence already existing." << endl;
516 delete res;
517 return kTRUE;
518 }
519 if (check==2)
520 rc=2;
521 else
522 {
523 cout << " - Deleting quasi-identical sequence " << atoi((*row)[0]) << endl;
524 if (DeleteSequence(atoi((*row)[0]))==2)
525 rc = 2;
526 }
527 }
528 else
529 {
530 TSQLRow *row=0;
531 while ((row=res->Next()))
532 {
533 cout << " - Deleting overlapping sequence " << atoi((*row)[0]) << endl;
534 if (DeleteSequence(atoi((*row)[0]))==2)
535 rc = 2;
536 }
537 }
538
539 delete res;
540
541 if (rc==2)
542 return 2;
543
544 if (!InsertSequence(runstart, runstop))
545 return 2;
546
547 return kTRUE;
548 }
549
550 Bool_t ReadResources(const char *fname)
551 {
552 TPRegexp regexp("^\\[.*\\]$");
553
554 ifstream fin(fname);
555 if (!fin)
556 {
557 cout << "Cannot open file " << fname << ": ";
558 cout << strerror(errno) << endl;
559 return kFALSE;
560 }
561
562 Int_t section = 0;
563
564 TString key;
565 while (1)
566 {
567 TString txt;
568 txt.ReadLine(fin);
569 if (!fin)
570 break;
571
572 txt = txt.Strip(TString::kBoth);
573
574 if (txt[0]=='#' || txt.IsNull())
575 continue;
576
577 if (txt[0]=='[' && section!=2)
578 {
579 //cout << txt << endl;
580 section = 0;
581 if (txt(regexp)=="[Transition]")
582 section = 1;
583 if (txt(regexp)=="[Regexp]")
584 section = 2;
585 continue;
586 }
587
588 TObjArray *arr = txt.Tokenize(" ");
589
590 if (arr->GetEntries()>0)
591 switch (section)
592 {
593 case 1:
594 {
595 TString key = arr->At(0)->GetName();
596 key.Prepend("f");
597 key.Append("KEY");
598
599 CheckList *list = dynamic_cast<CheckList*>(fMap.GetValue(key));
600 if (!list)
601 {
602 //cout << key << endl;
603 list = new CheckList;
604 fMap.Add(new TObjString(key), list);
605 }
606
607 if (arr->GetEntries()>1)
608 {
609 //cout << key << " ";
610 list->Add(new CheckMatch(*arr, 1));
611 }
612 }
613 break;
614 case 2:
615 fListRegexp.Add(new Rule(*arr));
616 break;
617 }
618
619 delete arr;
620 }
621
622 return kTRUE;
623 }
624
625 TString GetELT(const char *col, TSQLResult *res, TList &regexp)
626 {
627 TObjArray names; // array with old names (including regexp)
628
629 // Add to array and expand the array if necessary
630 TSQLRow *row=0;
631 while ((row=res->Next()))
632 names.AddAtAndExpand(new TObjString((*row)[1]), atoi((*row)[0]));
633
634 // Now a LUT is build which converts the keys for
635 // the names including the regexp into keys for
636 // the names excluding the regexp
637 TString elt(Form("ELT(RunData.f%sKEY+1", col));
638
639 // loop over all entries in the list
640 const Int_t n = names.GetSize();
641 for (int i=0; i<n; i++)
642 {
643 // For all entries which are not in the list
644 // write an undefined value into the LUT
645 TObject *o = names.UncheckedAt(i);
646 if (!o)
647 {
648 elt += ",0";
649 continue;
650 }
651
652 // Remove the regexp from the string which includes it
653 TString name = o->GetName();
654
655 TIter NextR(&regexp);
656 TObject *reg=0;
657 while ((reg=NextR()))
658 {
659 TPRegexp reg(reg->GetName());
660 const Ssiz_t pos = name.Index(reg, 0);
661 if (pos>0)
662 {
663 name.Remove(pos);
664 name += "-W";
665 break;
666 }
667 }
668
669 // Check if such a Key exists, if not insert it
670 const Int_t key = QueryKeyOfName(col, name);
671
672 // RESOLVE return code!!!
673 //if (key<0)
674 // return "";
675
676 // add index to the LUT
677 elt += Form(",%d", key);
678 }
679
680 // close LUT expression
681 // elt += ") AS Elt";
682 // elt += col;
683
684 elt += ") AS f";
685 elt += col;
686 elt += "KEY";
687
688 // return result
689 return elt;
690 }
691
692 TString GetELT(const char *col, TSQLResult *res, TString regexp)
693 {
694 TList list;
695 list.SetOwner();
696 list.Add(new TObjString(regexp));
697 return GetELT(col, res, list);
698 }
699
700 TString GetELTQuery(const char *col, const char *cond) const
701 {
702 return Form("SELECT RunData.f%sKEY, f%sName FROM RunData "
703 "LEFT JOIN %s ON RunData.f%sKEY=%s.f%sKEY "
704 "WHERE %s GROUP BY f%sName",
705 col, col, col, col, col, col, cond, col);
706 }
707
708 TString GetELTSource(const char *cond)
709 {
710 //query all sources observed in this night
711 TSQLResult *resx = Query(GetELTQuery("Source", cond));
712 if (!resx)
713 return "";
714
715 // In the case there is only a single source
716 // do not replace the source key by the ELT
717 if (resx->GetRowCount()==1)
718 return "fSourceKEY";
719
720 TString elts = GetELT("Source", resx, "\\-?W[1-9][ abc]?$");
721 delete resx;
722
723 return elts;
724 }
725
726 TString GetELTProject(const char *cond)
727 {
728 //query all project names observed in this night
729 TSQLResult *resx = Query(GetELTQuery("Project", cond));
730 if (!resx)
731 return "";
732
733 // In the case there is only a single project
734 // do not replace the project key by the ELT
735 if (resx->GetRowCount()==1)
736 return "fProjectKEY";
737
738 TList regexp;
739 regexp.Add(new TObjString("\\-?W[1-9][abc]?$"));
740 regexp.Add(new TObjString("\\-W[0-9]\\.[0-9][0-9]\\+[0-9][0-9][0-9]$"));
741
742 TString eltp2 = GetELT("Project", resx, regexp);
743 delete resx;
744 regexp.Delete();
745
746 return eltp2;
747 }
748
749 Bool_t HasAtLeastOne(TString src, TString chk) const
750 {
751 src.ToLower();
752 chk.ToLower();
753
754 for (int i=0; i<chk.Length(); i++)
755 if (src.First(chk[i])<0)
756 return kFALSE;
757
758 return kTRUE;
759 }
760
761 TString PrepareString(TSQLResult &res, TArrayI &runs)
762 {
763 // Number of result rows
764 const Int_t rows = res.GetRowCount();
765
766 runs.Set(rows); // initialize size of array for run numbers
767
768 TArrayD start(rows); // instantiate array for start times
769 TArrayD stop(rows); // instantiate array for stop times
770
771 TString str; // result string
772
773 Int_t idx=0;
774 TSQLRow *row=0;
775 while ((row=res.Next()))
776 {
777 runs[idx] = atoi((*row)[0]); // run number
778
779 const TString tstart = ((*row)[2]); // start time
780 const TString tstop = ((*row)[3]); // stop time
781
782 start[idx] = MTime(tstart).GetMjd(); // convert to double
783 stop[idx] = MTime(tstop).GetMjd(); // convert to double
784
785 // This is a workaround for broken start-times
786 if (tstart=="0000-00-00 00:00:00")
787 start[idx] = stop[idx];
788
789 // Add a run-type character for this run to the string
790 str += RunType((*row)[1]);
791
792 // Increase index
793 idx++;
794 }
795
796 // Now the P- and D- runs are classified by the time-distance
797 // to the next or previous C-Run
798 Double_t lastc = -1;
799 Double_t nextc = -1;
800 for (int i=0; i<str.Length(); i++)
801 {
802 if (str[i]=='C')
803 {
804 // Remember stop time of C-Run as time of last C-run
805 lastc = stop[i];
806
807 // Calculate start time of next C-Run
808 const TString residual = str(i+1, str.Length());
809 const Int_t pos = residual.First('C');
810
811 // No C-Run found anymore. Finished...
812 if (pos<0)
813 break;
814
815 // Remember start time of C-Run as time of next C-run
816 nextc = start[i+1+pos];
817 continue;
818 }
819
820 // Check whether the identifying character should
821 // be converted to lower case
822 if (start[i]-lastc>nextc-stop[i])
823 str[i] = tolower(str[i]);
824 }
825 //cout << str << endl;
826 return str;
827 }
828
829 void PrintResidual(Int_t runstart, Int_t runstop, TString residual, const char *descr)
830 {
831 residual.ToLower();
832
833 // Count number of unsequences "characters"
834 const Int_t nump = residual.CountChar('p');
835 const Int_t numd = residual.CountChar('d');
836 const Int_t numc = residual.CountChar('c');
837
838 // Print some information to the output steram
839 if (nump+numc+numd==0)
840 return;
841
842 cout << " ! " << runstart << "-" << runstop << " [" << setw(3) << 100*residual.Length()/(runstop-runstart+1) << "%]: " << descr << " not sequenced: P=" << nump << " C=" << numc << " D=" << numd;
843 if (numd>0)
844 cout << " (DATA!)";
845
846 if (nump==0 || numc==0 || numd==0)
847 cout << " Missing";
848 if (numd==0)
849 cout << " D";
850 if (nump==0)
851 cout << " P";
852 if (numc==0)
853 cout << " C";
854
855 cout << endl;
856 }
857
858 Int_t SplitBlock(Int_t runstart, Int_t runstop)
859 {
860 // Request data necessary to split block into sequences
861 const TString query=
862 Form("SELECT fRunNumber, fRunTypeKEY, fRunStart, fRunStop"
863 " FROM RunData "
864 " WHERE fRunNumber BETWEEN %d AND %d AND "
865 " fExcludedFDAKEY=1 AND (fRunTypeKEY BETWEEN 2 AND 4)"
866 " ORDER BY fRunNumber", runstart, runstop);
867
868 // Send query
869 TSQLResult *res = Query(query);
870 if (!res)
871 return 2;
872
873 // Get String containing the sequence of P-,C- and D-Runs
874 // and an array with the corresponding run-numbers
875 TArrayI runs;
876 const TString str = PrepareString(*res, runs);
877
878 delete res;
879
880 // Check if the prepared string at least contains one run of each type
881 if (!HasAtLeastOne(str, "PCD"))
882 {
883 PrintResidual(runstart, runstop, str, "Block");
884 return kTRUE;
885 }
886
887 // Start the sequence building
888 // ---------------------------
889
890 Int_t pos = 0; // Position in the string
891 TExMap map; // Map for resulting sequences
892 TString residual; // Unsequences "characters"
893
894 // Step through the string one character by one
895 cout << " ";
896 while (pos<str.Length())
897 {
898 Bool_t found = kFALSE;
899
900 // Loop over the predefined regular expressions
901 TIter Next(&fListRegexp);
902 Rule *obj = 0;
903 while ((obj=(Rule*)Next()))
904 {
905 // Check if regular expressions matchs
906 const Ssiz_t len = obj->Match(str, pos, runs[pos]);
907 if (len>0)
908 {
909 // In case of match get the substring which
910 // defines the sequence and print it
911 TString sub = str(pos, len);
912 sub.ToUpper();
913 cout << runs[pos]<<":"<< sub << "|";
914
915 // Add the first and last run of the sequence to the map
916 map.Add(runs[pos], runs[pos+len-1]);
917
918 // A sequence was found...
919 found = kTRUE;
920
921 // step forward in the string by the length of the sequence
922 pos += len;
923 break;
924 }
925 }
926
927 // If a sequence was found go on...
928 if (found)
929 continue;
930
931 // print unsequenced characters
932 cout << (char)tolower(str[pos]);
933
934 // Count the number of "characters" not sequenced
935 residual += str[pos];
936
937 // step one character forward
938 pos++;
939 }
940 cout << endl;
941
942 PrintResidual(runstart, runstop, residual, "Runs ");
943
944 // Create all sequences which were previously found
945 Long_t first, last;
946 TExMapIter iter(&map);
947 while (iter.Next(first, last))
948 if (CreateSequence(first, last)==2)
949 return 2;
950
951 return kTRUE;
952 }
953
954 Char_t RunType(Int_t n) const
955 {
956 switch (n)
957 {
958 case 2: return 'D';
959 case 3: return 'P';
960 case 4: return 'C';
961 }
962 return '-';
963 }
964 Char_t RunType(const char *str) const
965 {
966 return RunType(atoi(str));
967 }
968
969 Int_t BuildBlocks(TSQLResult *res, TExMap &blocks)
970 {
971 // col key content
972 // -----------------------------
973 // 0 - runnumber
974 // 1 0 RunTypeKEY
975 // 2 1 source
976 // 3 2 project
977 // . .
978 // . .
979 // . . from transition.txt
980 //
981
982 //build blocks of runs, which have the same values
983 //for each block the first and the last run are stored in a TExMap
984 //the values are checked with the help of an array of TStrings
985 Long_t runstart = -1;
986 Long_t runstop = -1;
987
988 const UInt_t ncols = res->GetFieldCount();
989
990 TString keys[ncols]; // Index 0 is not used (Index 1: RunType)
991
992 // Loop over runs
993 TSQLRow *row=0;
994 while ((row=res->Next()))
995 {
996 // This is the runnumber of the first run in the new block
997 if (runstart<0)
998 runstart=atoi((*row)[0]);
999
1000 // Check which transitions might be allowed for this run and
1001 // which are not. Check also which attributes should be ignored
1002 // for this run. The information about it is stored in the map.
1003 Int_t rc[ncols];
1004 for (UInt_t i=2; i<ncols; i++)
1005 rc[i] = CheckTransition(*res, keys, *row, i);
1006
1007 for (UInt_t i=2; i<ncols; i++)
1008 {
1009 switch (rc[i])
1010 {
1011 case kTRUE: // Transition allowed, switch to new attribute
1012 keys[i] = (*row)[i];
1013 continue;
1014
1015 case -1: // Ignore attribute, continue with next one
1016 //if (keys[i] == (*row)[i])
1017 // break; // Only ignore it if it is really different
1018 continue;
1019
1020 case kFALSE: // No decision (nothing matching found in map)
1021 break; // go on checking
1022
1023 }
1024
1025 // If past-attribute is not yet initialized, do it now.
1026 if (keys[i].IsNull())
1027 keys[i] = (*row)[i];
1028
1029 // Check whether key has changed for this run
1030 // (this condition is never true for the first run)
1031 // This condition in only reached if keys[i] was initialized
1032 if (keys[i] == (*row)[i])
1033 continue;
1034
1035 // Found one block with unique keys, fill values into TExMap
1036 // (except if this is the first run processed)
1037 cout << endl;
1038 cout << " - Identical conditions from " << runstart << " to " << runstop << endl;
1039 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1040 if (SplitBlock(runstart, runstop)==2)
1041 return 2;
1042 cout << " - Transition from " << RunType(keys[1]) << ":";
1043 cout << QueryNameOfKey(res->GetFieldName(i), keys[i]) << " <" << keys[i] << "> to " << RunType((*row)[1]) << ":";
1044 cout << QueryNameOfKey(res->GetFieldName(i), (*row)[i]) << " <" << (*row)[i] << ">";
1045 cout << " in " << res->GetFieldName(i) << " of " << (*row)[0] << endl;
1046
1047 // This is already the first run of the new block
1048 runstart=atoi((*row)[0]);
1049
1050 // These are the keys corresponding to the first run
1051 // in the new block. All following runs should have
1052 // identical keys. Do not set the attribute if for
1053 // this attribute the transition check evaluated
1054 // to "ignore".
1055 for (UInt_t j=2; j<ncols; j++)
1056 keys[j] = rc[j]==-1 ? "" : (*row)[j];
1057
1058 break;
1059 }
1060
1061 keys[1] = (*row)[1]; // Remember run type for next run
1062
1063 // This is the new runnumber of the last run in this block
1064 runstop=atoi((*row)[0]);
1065 }
1066
1067 // If runstart==-1 a possible block would contain a single run...
1068 if (runstart>0 && runstart!=runstop)
1069 {
1070 cout << " - Identical conditions from " << runstart << " to " << runstop << " (last run)" << endl;
1071 //fill values into TExMap (last value)
1072 blocks.Add((ULong_t)blocks.GetSize(), runstart, runstop);
1073 if (SplitBlock(runstart, runstop)==2)
1074 return 2;
1075 }
1076 cout << "Done." << endl << endl;
1077
1078 return kTRUE;
1079 }
1080
1081public:
1082 SequenceBuild(TEnv &env) : MSQLMagic(env)
1083 {
1084 cout << "buildsequences" << endl;
1085 cout << "--------------" << endl;
1086 cout << endl;
1087 cout << "Connected to " << GetName() << endl;
1088
1089 fListRegexp.SetOwner();
1090
1091 // FIXME: THIS IS NOT YET HANDLED
1092 if (ReadResources("resources/sequences.rc"))
1093 return;
1094 }
1095 SequenceBuild()
1096 {
1097 fMap.DeleteAll();
1098 }
1099
1100 void SetPathRawData(const char *path) { fPathRawData=path; }
1101 void SetPathSequences(const char *path) { fPathSequences=path; }
1102
1103 int Build(TString day)
1104 {
1105 cout << endl;
1106 cout << "Night of sunrise at " << day << ":" << endl;
1107
1108 day += " 13:00:00";
1109 const TString cond =
1110 Form("(fRunStart>ADDDATE(\"%s\", INTERVAL -1 DAY) AND fRunStart<\"%s\") "
1111 "AND fExcludedFDAKEY=1 AND fRunTypeKEY BETWEEN 2 AND 4 ",
1112 day.Data(), day.Data());
1113
1114 //query all sources observed in this night
1115 const TString elts = GetELTSource(cond);
1116 if (elts.IsNull())
1117 return 2;
1118
1119 //query all project names observed in this night
1120 const TString eltp2 = GetELTProject(cond);
1121 if (elts.IsNull())
1122 return 2;
1123
1124 // Setup query to get all values from the database,
1125 // that are relevant for building sequences
1126 TString query("SELECT fRunNumber, fRunTypeKEY, ");
1127 query += elts;
1128 query += ", ";
1129 query += eltp2;
1130
1131 // Now add all entries from the transition table to the query
1132 TIter NextPair(&fMap);
1133 TObject *mapkey = 0;
1134 while ((mapkey=(TPair*)NextPair()))
1135 if (!query.Contains(mapkey->GetName()))
1136 {
1137 query += ", ";
1138 query += mapkey->GetName();
1139 }
1140 query += Form(" FROM RunData WHERE %s ORDER BY fRunNumber", cond.Data());
1141
1142 TSQLResult *res = Query(query);
1143 if (!res)
1144 return 2;
1145
1146 TExMap blocks;
1147 const Int_t rc = BuildBlocks(res, blocks);
1148 delete res;
1149
1150 return rc;
1151 }
1152
1153 Int_t Build()
1154 {
1155 //get all dates from the database
1156 TSQLResult *res = Query("SELECT fDate FROM SequenceBuildStatus ORDER BY fDate DESC");
1157 if (!res)
1158 return 2;
1159
1160 //execute buildsequenceentries for all dates
1161 TSQLRow *row=0;
1162 while ((row=res->Next()))
1163 Build((*row)[0]);
1164
1165 delete res;
1166
1167 return 1;
1168 }
1169 ClassDef(SequenceBuild, 0)
1170};
1171
1172ClassImp(SequenceBuild);
1173
1174int buildsequenceentries(TString day, TString datapath, TString sequpath, Bool_t dummy=kTRUE)
1175{
1176 TEnv env("sql.rc");
1177
1178 SequenceBuild serv(env);
1179 if (!serv.IsConnected())
1180 {
1181 cout << "ERROR - Connection to database failed." << endl;
1182 return 0;
1183 }
1184
1185 serv.SetIsDummy(dummy);
1186 serv.SetPathRawData(datapath);
1187 serv.SetPathSequences(sequpath);
1188 return serv.Build(day);
1189}
1190
1191//
1192// Build Sequences for all Nights
1193//
1194int buildsequenceentries(TString datapath, TString sequpath, Bool_t dummy=kTRUE)
1195{
1196 TEnv env("sql.rc");
1197
1198 SequenceBuild serv(env);
1199 if (!serv.IsConnected())
1200 {
1201 cout << "ERROR - Connection to database failed." << endl;
1202 return 0;
1203 }
1204
1205 serv.SetIsDummy(dummy);
1206 serv.SetPathRawData(datapath);
1207 serv.SetPathSequences(sequpath);
1208 return serv.Build();
1209}
1210
1211int buildsequenceentries(Bool_t dummy=kTRUE)
1212{
1213 return buildsequenceentries("", "", dummy);
1214}
1215
1216int buildsequenceentries(TString day, Bool_t dummy=kTRUE)
1217{
1218 return buildsequenceentries(day, "", "", dummy);
1219}
Note: See TracBrowser for help on using the repository browser.