Context Navigation

source: trunk/MagicSoft/Mars/datacenter/macros/filldotrun.C@ 8006

Visit:

Last change on this file since 8006 was 7961, checked in by Daniela Dorner, 18 years ago
* empty log message *
File size: 21.4 KB

Line
1	/* ======================================================================== *\
2	!
3	! *
4	! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5	! * Software. It is distributed to you in the hope that it can be a useful
6	! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7	! * It is distributed WITHOUT ANY WARRANTY.
8	! *
9	! * Permission to use, copy, modify and distribute this software and its
10	! * documentation for any purpose is hereby granted without fee,
11	! * provided that the above copyright notice appear in all copies and
12	! * that both that copyright notice and this permission notice appear
13	! * in supporting documentation. It is provided "as is" without express
14	! * or implied warranty.
15	! *
16	!
17	!
18	! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
19	! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
20	!
21	! Copyright: MAGIC Software Development, 2000-2006
22	!
23	!
24	\* ======================================================================== */
25
26	/////////////////////////////////////////////////////////////////////////////
27	//
28	// filldotrun.C
29	// ============
30	//
31	// This macro is used in the datacenter to automatically fill the run-database
32	// with the information stored in the .run-files written by the central
33	// control.
34	//
35	// To following Arehucas versions are Currently supported:
36	// 040505-0, 040514-0,
37	// 040518-0, 040727-0,
38	// 041113-0, 041209-0, 041221-0
39	// 050224-0, 050317-0, 050322-0, 050401-0, 050413-0, 050415-0, 050714-0,
40	// 050719-0, 050829-0, 051025-0,
41	// 060330-0, 060401-0, 060808-0
42	//
43	// Usage:
44	// .x filldotrun.C+("/data/MAGIC/Period019/ccdata", kTRUE)
45	//
46	// While the first argument is the directory in which all subdirectories where
47	// searches for CC_*.run files. All these files were analysed and the run
48	// info will be put into the DB, eg:
49	// "/magic/subsystemdata/cc" would do it for all data
50	// "/magic/subsystemdata/cc/2005" for one year
51	// "/magic/subsystemdata/cc/2005/11" for one month
52	// "/magic/subsystemdata/cc/2005/11/11" for a single day
53	// "/magic/subsystemdata/cc/2005/11/11/file.run" for a single file
54	//
55	// The second argument is the 'dummy-mode'. If it is kTRUE dummy-mode is
56	// switched on and nothing will be written into the database. Instead
57	// informations about the subtables are displayed. This is usefull for tests
58	// when adding a new arehucas version support. If it is kFALSE the information
59	// are written into the subtables and the runs info is written into the
60	// rundatabase.
61	//
62	// In the automatic case it makes sense to check the logfiles to make sure
63	// that everything is fine...
64	//
65	// Make sure, that database and password are corretly set in a resource
66	// file called sql.rc and the resource file is found.
67	//
68	// Remark: Running it from the commandline looks like this:
69	// root -q -l -b filldotrun.C+\(\"path\"\,kFALSE\) 2>&1 \| tee filldotrun.log
70	//
71	// Returns 0 in case of failure and 1 in case of success.
72	//
73	/////////////////////////////////////////////////////////////////////////////
74	#include <iostream>
75	#include <iomanip>
76	#include <fstream>
77
78	#include <TEnv.h>
79	#include <TMath.h>
80	#include <TRegexp.h>
81
82	#include <TSQLRow.h>
83	#include <TSQLResult.h>
84
85	#include "MTime.h"
86	#include "MDirIter.h"
87	#include "MSQLServer.h"
88
89	using namespace std;
90
91	Bool_t ExistStr(MSQLServer &serv, const char column, const char table, const char *test)
92	{
93	TString query(Form("SELECT %s FROM %s WHERE %s='%s'", column, table, column, test));
94	TSQLResult *res = serv.Query(query);
95	if (!res)
96	return kFALSE;
97
98	Bool_t rc = kFALSE;
99
100	TSQLRow *row=res->Next();
101	if (row && (*row)[0])
102	rc=kTRUE;
103
104	delete res;
105	return rc;
106	}
107
108	//get key for the value
109	// if value is not existing, insert value and get key
110	Int_t QueryNameKEY(MSQLServer &serv, Bool_t dummy, const char col, const char name, Bool_t insert=kTRUE)
111	{
112	TString query;
113
114	query = Form("SELECT f%sKEY FROM %s WHERE f%sName='%s'", col, col, col, name);
115	TSQLResult *res = serv.Query(query);
116	if (!res)
117	return -1;
118
119	TSQLRow *row=res->Next();
120
121	Int_t rc = row && (row)[0] ? atoi((row)[0]) : -1;
122
123	delete res;
124
125	if (rc>=0)
126	return rc;
127
128	if (!insert)
129	return -1;
130
131	//insert new value
132	query = Form("INSERT %s (f%sName) VALUES (\"%s\");", col, col, name);
133
134	if (dummy)
135	{
136	cout << query << endl;
137	return 0;
138	}
139
140	res=serv.Query(query);
141	if (!res)
142	return -1;
143
144	delete res;
145
146	Int_t key = QueryNameKEY(serv, dummy, col, name, kFALSE);
147	if (key>0)
148	{
149	cout << "New " << col << ": " << name << endl;
150	return key;
151	}
152
153	cout << "ERROR: " << query << endl;
154	return kFALSE;
155	}
156
157
158	Int_t insert(MSQLServer &serv, Bool_t dummy, TString filename)
159	{
160	ifstream fin(filename);
161	if (!fin)
162	{
163	cout << "Could not open file " << filename << endl;
164	return -1;
165	}
166
167	TString strng;
168	strng.ReadLine(fin);
169	if (strng!=TString("[CC Plain Run Summary File]"))
170	{
171	cout << filename << ": No Plain Run Summary File" << endl;
172	cout << "First Line: " << strng << endl;
173	cout << endl;
174	return -1;
175	}
176
177	strng.ReadLine(fin);
178	TRegexp reg("[0-9][0-9][0-9][0-9][0-9][0-9]-[0-9]");
179	TString arehucas = strng(reg);
180	arehucas.Prepend("20");
181	arehucas.ReplaceAll("-", "");
182
183	Int_t version = atoi(arehucas.Data());
184	if (version!=200405050 && version!=200405140 && version!=200405180 &&
185	version!=200407270 && version!=200411130 && version!=200412090 &&
186	version!=200412210 &&
187	version!=200502240 && version!=200503170 && version!=200503220 &&
188	version!=200504010 && version!=200504130 && version!=200504150 &&
189	version!=200507140 && version!=200507190 && version!=200508290 &&
190	version!=200510250 &&
191	version!=200603300 && version!=200604010 && version!=200608080 )
192	{
193	cout << filename << ": File Version unknown - please update the macro!" << endl;
194	cout << "Second Line: " << strng << endl;
195	cout << endl;
196	return -1;
197	}
198
199	cout << " V" << version << " " << flush;
200
201	Int_t cnt=0;
202	while (1)
203	{
204	// ========== Col 1: Run Number =========
205	//Reading the line
206	//and converting some strings to ints/floats
207	strng.ReadToDelim(fin, ' ');
208	if (!fin)
209	break;
210
211	Int_t runnumber = atoi(strng.Data());
212
213	//runnumber=0 means no valid dataset
214	//-> continue
215	if (runnumber == 0)
216	{
217	strng.ReadLine(fin);
218	cout << "Runnumber == 0" << endl;
219	continue;
220	}
221
222	//cout << "RunNo: " << runnumber << " ";
223
224	if (ExistStr(serv, "fRunNumber", "RunData", strng.Data()))
225	{
226	// FIXME: Maybe we can implement an switch to update mode?
227	cout << "Run #" << runnumber << " already existing... skipped." << endl;
228	strng.ReadLine(fin);
229	continue;
230	}
231
232	// ========== Col 2: Run Type =========
233	strng.ReadToDelim(fin, ' ');
234	if (strng.Contains("???"))
235	strng="n/a";
236
237	Int_t runtype = QueryNameKEY(serv, dummy, "RunType", strng.Data(), kFALSE);
238	if (runtype<0)
239	{
240	cout << "ERROR - RunType " << strng << " not available." << endl;
241	strng.ReadLine(fin);
242	continue;
243	}
244
245	//cout << runtype << " ";
246
247	// ========== Col 3,4: Start Time =========
248	TString startdate, starttime;
249	startdate.ReadToDelim(fin, ' ');
250	starttime.ReadToDelim(fin, ' ');
251	//cout << startdate << " " << starttime << " ";
252
253	// ========== Col 5,6: Stop Time =========
254	TString stopdate, stoptime;
255	stopdate.ReadToDelim(fin, ' ');
256	stoptime.ReadToDelim(fin, ' ');
257	//cout << stopdate << " " << stoptime << " ";
258
259	if (startdate.Contains("???"))
260	startdate="0000-00-00";
261	if (starttime.Contains("???"))
262	starttime="00:00:00";
263	if (stopdate.Contains("???"))
264	stopdate="0000-00-00";
265	if (stoptime.Contains("???"))
266	stoptime="00:00:00";
267
268	// ========== Col 7: Source Name =========
269	strng.ReadToDelim(fin, ' ');
270	if (strng.Contains("???"))
271	strng="Unavailable";
272
273	Int_t sourcekey = QueryNameKEY(serv, dummy, "Source", strng.Data());
274	if (sourcekey<0)
275	{
276	strng.ReadLine(fin);
277	continue;
278	}
279	//cout << sourcekey << " ";
280
281	// ========== Col 8,9: Local source position =========
282	strng.ReadToDelim(fin, ' ');
283	Float_t zd = atof(strng.Data());
284
285	strng.ReadToDelim(fin, ' ');
286	Float_t az = atof(strng.Data());
287
288	//cout << zd << " " << az << " ";
289
290	// ========== Col 10: Number of Events =========
291	strng.ReadToDelim(fin, ' ');
292	Int_t evtno = atoi(strng.Data());
293
294	//cout << evtno << " ";
295
296	// ========== Col 11: Project Name =========
297	strng.ReadToDelim(fin, ' ');
298	if (strng.Contains("???"))
299	strng="Unavailable";
300
301	Int_t projkey = QueryNameKEY(serv, dummy, "Project", strng.Data());
302	if (projkey<0)
303	{
304	strng.ReadLine(fin);
305	continue;
306	}
307	//cout << projkey << " ";
308
309	// ========== Col 12: Trigger Table Name =========
310	// starting from version 200411130: Col 12,13: Trigger Table Name =========
311	strng.ReadToDelim(fin, ' ');
312	if (strng.Contains("???"))
313	strng="n/a";
314
315	Int_t l1triggerkey=1;
316	Int_t l2triggerkey=1;
317	if (version >=200411130)
318	{
319	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", strng.Data());
320	if (l1triggerkey<0)
321	{
322	strng.ReadLine(fin);
323	continue;
324	}
325
326	strng.ReadToDelim(fin, ' ');
327	if (strng.Contains("???"))
328	strng="n/a";
329
330	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", strng.Data());
331	if (l2triggerkey<0)
332	{
333	strng.ReadLine(fin);
334	continue;
335	}
336	}
337	else
338	{
339	Int_t c=0;
340
341	if (strng.Contains(":"))
342	c=1;
343
344	if (strng.Contains("L1_") && !(strng.Contains(":")))
345	c=2;
346
347	if (strng.Contains("n/a"))
348	c=3;
349
350	switch (c)
351	{
352	case 0:
353	{
354	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", strng.Data());
355	if (l2triggerkey<0)
356	{
357	strng.ReadLine(fin);
358	continue;
359	}
360
361	strng="n/a";
362	l1triggerkey = 1;
363
364	break;
365	}
366	case 1:
367	{
368	TString L1TT, L2TT;
369	L2TT=strng(7,12);
370	L1TT=strng(0,6);
371
372	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", L1TT.Data());
373	if (l1triggerkey<0)
374	{
375	strng.ReadLine(fin);
376	continue;
377	}
378
379	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", L2TT.Data());
380	if (l2triggerkey<0)
381	{
382	strng.ReadLine(fin);
383	continue;
384	}
385
386	break;
387	}
388	case 2:
389	{
390	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", strng.Data());
391	if (l1triggerkey<0)
392	{
393	strng.ReadLine(fin);
394	continue;
395	}
396
397	strng="n/a";
398	l2triggerkey = 1;
399
400	break;
401	}
402	case 3:
403	{
404	l1triggerkey = 1;
405	l2triggerkey = 1;
406	break;
407	}
408	default:
409	{
410	cout << "WARNING: neither L1 nor L2 Trigger table - please check what is happening." << strng << endl;
411	break;
412	}
413	}
414	}
415
416	// ========== Col 13-15: TrigRate, L2 UnPresc Rate, L2 Presc Rate ==========
417	strng.ReadToDelim(fin, ' ');
418	Float_t trigrate = atof(strng.Data());
419
420	strng.ReadToDelim(fin, ' ');
421	Float_t l2uprate = atof(strng.Data());
422
423	strng.ReadToDelim(fin, ' ');
424	Float_t l2prrate = atof(strng.Data());
425
426	// ========== Col 16,17: DaqRate, Storage Rate ==========
427	strng.ReadToDelim(fin, ' ');
428	Float_t daqrate = atof(strng.Data());
429
430	strng.ReadToDelim(fin, ' ');
431	Float_t storerate = atof(strng.Data());
432
433	// ========== Col 18: HV table =========
434	if (version==200405050 \|\| version==200405140)
435	strng.ReadToDelim(fin, '\n');
436	else
437	strng.ReadToDelim(fin, ' ');
438	if (strng.Contains("???"))
439	strng="n/a";
440
441	Int_t hvkey = QueryNameKEY(serv, dummy, "HvSettings", strng.Data());
442	if (hvkey<0)
443	{
444	//strng.ReadLine(fin);
445	continue;
446	}
447
448	if (version==200405180 \|\| version==200407270)
449	strng.ReadLine(fin);
450
451	Int_t testflagkey=1;
452	Int_t lightcondkey=1;
453	Int_t dttablekey=1;
454	Int_t triggerdelaytablekey=1;
455	Int_t calibrationscriptkey=1;
456	if (version==200411130 \|\| version==200412090 \|\| version==200412210
457	\|\| version==200502240 \|\| version==200503170 \|\| version==200503220
458	\|\| version==200504010 \|\| version==200504130 \|\| version==200504150
459	\|\| version==200507140 \|\| version==200507190 \|\| version==200508290
460	\|\| version==200510250 \|\| version==200603300 \|\| version==200604010
461	\|\| version==200608080 )
462	{
463	// ========== Col 19-35: DC and HV-values, mjd =========
464	for (int i=0 ; i<17 ; i++)
465	{
466	strng.ReadToDelim(fin, ' ');
467	}
468
469	// ========== Col 36: test-flag =========
470	strng.ReadToDelim(fin, ' ');
471	if (strng.Contains("???"))
472	strng="n/a";
473
474	testflagkey = QueryNameKEY(serv, dummy, "TestFlag", strng.Data());
475	if (testflagkey<0)
476	{
477	strng.ReadLine(fin);
478	continue;
479	}
480
481	// ========== Col 37: light conditions =========
482	strng.ReadToDelim(fin, ' ');
483	if (strng.Contains("???"))
484	strng="n/a";
485
486	lightcondkey = QueryNameKEY(serv, dummy, "LightConditions", strng.Data());
487	if (lightcondkey<0)
488	{
489	strng.ReadLine(fin);
490	continue;
491	}
492
493	// ========== Col 38: discriminator threshold table =========
494	strng.ReadToDelim(fin, ' ');
495	if (strng.Contains("???"))
496	strng="n/a";
497
498	dttablekey = QueryNameKEY(serv, dummy, "DiscriminatorThresholdTable", strng.Data());
499	if (dttablekey<0)
500	{
501	strng.ReadLine(fin);
502	continue;
503	}
504
505	// ========== Col 39: trigger delay table =========
506	strng.ReadToDelim(fin, ' ');
507	if (strng.Contains("???"))
508	strng="n/a";
509
510	triggerdelaytablekey = QueryNameKEY(serv, dummy, "TriggerDelayTable", strng.Data());
511	if (triggerdelaytablekey<0)
512	{
513	strng.ReadLine(fin);
514	continue;
515	}
516
517	// ========== Col 40,41: Telescope RA and Dec sent to drive =========
518	strng.ReadToDelim(fin, ' ');
519	strng.ReadToDelim(fin, ' ');
520
521	// ========== Col 42: Calibration Script =========
522	if (version==200411130 \|\| version==200412090 \|\| version==200412210
523	\|\| version==200502240 \|\| version==200503170 \|\| version==200503220
524	\|\| version==200504010 \|\| version==200504130 \|\| version==200504150
525	\|\| version==200507140 \|\| version==200507190 \|\| version==200508290
526	\|\| version==200510250)
527	strng.ReadToDelim(fin, '\n');
528	else
529	strng.ReadToDelim(fin, ' ');
530	if (strng.Contains("???"))
531	strng="n/a";
532
533	calibrationscriptkey = QueryNameKEY(serv, dummy, "CalibrationScript", strng.Data());
534	if (calibrationscriptkey<0)
535	{
536	strng.ReadLine(fin);
537	continue;
538	}
539
540	}
541
542	Int_t observationmodekey=1;
543	if (version==200603300 \|\| version==200604010 \|\| version==200608080)
544	{
545	// ========== Col 43: Observation Mode =========
546	strng.ReadToDelim(fin, ' ');
547	if (strng.Contains("???"))
548	strng="n/a";
549
550	observationmodekey = QueryNameKEY(serv, dummy, "ObservationMode", strng.Data());
551	if (observationmodekey<0)
552	{
553	strng.ReadLine(fin);
554	continue;
555	}
556
557	// ========== Col 44-51: Source RA and Dec, DT's and IPR =========
558	for (int i=0 ; i<7 ; i++)
559	{
560	strng.ReadToDelim(fin, ' ');
561	}
562	strng.ReadToDelim(fin, '\n');
563	}
564
565
566	// ================================================================
567	// ========== Data read from file now access the database =========
568	// ================================================================
569
570	//assemlbe the query that is needed to insert the values of this run
571	TString query;
572	query += "INSERT RunData SET ";
573
574	query += Form("fRunNumber=%d, ", runnumber);
575	query += Form("fRunTypeKEY=%d, ", runtype);
576	query += Form("fProjectKEY=%d, ", projkey);
577	query += Form("fSourceKEY=%d, ", sourcekey);
578	query += Form("fNumEvents=%d, ", evtno);
579	query += Form("fRunStart=\"%s %s\", ", startdate.Data(), starttime.Data());
580	query += Form("fRunStop=\"%s %s\", ", stopdate.Data(), stoptime.Data());
581	query += Form("fL1TriggerTableKEY=%d, ", l1triggerkey);
582	query += Form("fL2TriggerTableKEY=%d, ", l2triggerkey);
583	query += Form("fTestFlagKEY=%d, ", testflagkey);
584	query += Form("fCalibrationScriptKEY=%d, ", calibrationscriptkey);
585	query += Form("fTriggerDelayTableKEY=%d, ", triggerdelaytablekey);
586	query += Form("fDiscriminatorThresholdTableKEY=%d, ", dttablekey);
587	query += Form("fLightConditionsKEY=%d, ", lightcondkey);
588	query += Form("fHvSettingsKEY=%d, ", hvkey);
589	query += Form("fObservationModeKEY=%d, ", observationmodekey);
590	if (!TMath::IsNaN(zd) && TMath::Finite(zd))
591	query += Form("fZenithDistance=%d, ", TMath::Nint(zd));
592	if (!TMath::IsNaN(az) && TMath::Finite(az))
593	query += Form("fAzimuth=%d, ", TMath::Nint(az));
594	if (!TMath::IsNaN(storerate) && TMath::Finite(storerate))
595	query += Form("fDaqStoreRate=%d, ", TMath::Nint(storerate));
596	if (!TMath::IsNaN(daqrate) && TMath::Finite(daqrate))
597	query += Form("fDaqTriggerRate=%d, ", TMath::Nint(daqrate));
598	if (!TMath::IsNaN(trigrate) && TMath::Finite(trigrate))
599	query += Form("fMeanTriggerRate=%d, ", TMath::Nint(trigrate));
600	if (!TMath::IsNaN(l2prrate) && TMath::Finite(l2prrate))
601	query += Form("fL2RatePresc=%d, ", TMath::Nint(l2prrate));
602	if (!TMath::IsNaN(l2uprate) && TMath::Finite(l2uprate))
603	query += Form("fL2RateUnpresc=%d, ", TMath::Nint(l2uprate));
604	query += "fMagicNumberKEY=1, fExcludedFDAKEY=1";
605
606	cnt++;
607
608	if (dummy)
609	continue;
610
611	//send query, add dataset to DB
612	TSQLResult *res = serv.Query(query);
613	if (!res)
614	return -1;
615	delete res;
616
617	//create entry in table RunProcessStatus for this runnumber
618	TString query2=Form("INSERT RunProcessStatus SET fRunNumber=%d, fTimingCorrection='1970-01-01 00:00:00'",
619	runnumber);
620	if (testflagkey==3)
621	query2+=" , fDataCheckDone='1970-01-01 00:00:00'";
622	res = serv.Query(query2);
623	if (!res)
624	return -1;
625	delete res;
626	}
627
628	return cnt;
629
630	}
631
632	// This tool will work from Period017 (2004_05_17) on...
633	int filldotrun(const TString path="/data/MAGIC/Period018/ccdata", Bool_t dummy=kTRUE)
634	{
635	TEnv env("sql.rc");
636
637	MSQLServer serv(env);
638	if (!serv.IsConnected())
639	{
640	cout << "ERROR - Connection to database failed." << endl;
641	return 0;
642	}
643	cout << "filldotrun" << endl;
644	cout << "----------" << endl;
645	cout << endl;
646	cout << "Connected to " << serv.GetName() << endl;
647	cout << "Search Path: " << path << endl;
648	cout << endl;
649
650	if (path.EndsWith(".run"))
651	{
652	cout << path(TRegexp("CC_.*.run", kFALSE)) << flush;
653	Int_t n = insert(serv, dummy, path);
654	cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
655
656	return n<0 ? 2 : 1;
657	}
658
659	MDirIter Next(path, "CC_*.run", -1);
660	while (1)
661	{
662	TString name = Next();
663	if (name.IsNull())
664	break;
665
666	cout << name(TRegexp("CC_.*.run", kFALSE)) << flush;
667	Int_t n = insert(serv, dummy, name);
668	cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
669
670	if (n<0)
671	return 2;
672	}
673
674	return 1;
675	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: