Context Navigation

source: trunk/MagicSoft/Mars/datacenter/macros/filldotrun.C@ 8947

Visit:

Last change on this file since 8947 was 8881, checked in by hoehne, 17 years ago
* empty log message *
File size: 21.6 KB

Line
1	/* ======================================================================== *\
2	!
3	! *
4	! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5	! * Software. It is distributed to you in the hope that it can be a useful
6	! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7	! * It is distributed WITHOUT ANY WARRANTY.
8	! *
9	! * Permission to use, copy, modify and distribute this software and its
10	! * documentation for any purpose is hereby granted without fee,
11	! * provided that the above copyright notice appear in all copies and
12	! * that both that copyright notice and this permission notice appear
13	! * in supporting documentation. It is provided "as is" without express
14	! * or implied warranty.
15	! *
16	!
17	!
18	! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
19	! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
20	!
21	! Copyright: MAGIC Software Development, 2000-2006
22	!
23	!
24	\* ======================================================================== */
25
26	/////////////////////////////////////////////////////////////////////////////
27	//
28	// filldotrun.C
29	// ============
30	//
31	// This macro is used in the datacenter to automatically fill the run-database
32	// with the information stored in the .run-files written by the central
33	// control.
34	//
35	// To following Arehucas versions are Currently supported:
36	// 040505-0, 040514-0,
37	// 040518-0, 040727-0,
38	// 041113-0, 041209-0, 041221-0
39	// 050224-0, 050317-0, 050322-0, 050401-0, 050413-0, 050415-0, 050714-0,
40	// 050719-0, 050829-0, 051025-0,
41	// 060330-0, 060401-0, 060808-0
42	// 070416-0,
43	// 080220-0
44	//
45	// Usage:
46	// .x filldotrun.C+("/data/MAGIC/Period019/ccdata", kTRUE)
47	//
48	// While the first argument is the directory in which all subdirectories where
49	// searches for CC_*.run files. All these files were analysed and the run
50	// info will be put into the DB, eg:
51	// "/magic/subsystemdata/cc" would do it for all data
52	// "/magic/subsystemdata/cc/2005" for one year
53	// "/magic/subsystemdata/cc/2005/11" for one month
54	// "/magic/subsystemdata/cc/2005/11/11" for a single day
55	// "/magic/subsystemdata/cc/2005/11/11/file.run" for a single file
56	//
57	// The second argument is the 'dummy-mode'. If it is kTRUE dummy-mode is
58	// switched on and nothing will be written into the database. Instead
59	// informations about the subtables are displayed. This is usefull for tests
60	// when adding a new arehucas version support. If it is kFALSE the information
61	// are written into the subtables and the runs info is written into the
62	// rundatabase.
63	//
64	// In the automatic case it makes sense to check the logfiles to make sure
65	// that everything is fine...
66	//
67	// Make sure, that database and password are corretly set in a resource
68	// file called sql.rc and the resource file is found.
69	//
70	// Remark: Running it from the commandline looks like this:
71	// root -q -l -b filldotrun.C+\(\"path\"\,kFALSE\) 2>&1 \| tee filldotrun.log
72	//
73	// Returns 0 in case of failure and 1 in case of success.
74	//
75	/////////////////////////////////////////////////////////////////////////////
76	#include <iostream>
77	#include <iomanip>
78	#include <fstream>
79
80	#include <TEnv.h>
81	#include <TMath.h>
82	#include <TRegexp.h>
83
84	#include <TSQLRow.h>
85	#include <TSQLResult.h>
86
87	#include "MTime.h"
88	#include "MDirIter.h"
89	#include "MSQLServer.h"
90
91	using namespace std;
92
93	Bool_t ExistStr(MSQLServer &serv, const char column, const char table, const char *test)
94	{
95	TString query(Form("SELECT %s FROM %s WHERE %s='%s'", column, table, column, test));
96	TSQLResult *res = serv.Query(query);
97	if (!res)
98	return kFALSE;
99
100	Bool_t rc = kFALSE;
101
102	TSQLRow *row=res->Next();
103	if (row && (*row)[0])
104	rc=kTRUE;
105
106	delete res;
107	return rc;
108	}
109
110	//get key for the value
111	// if value is not existing, insert value and get key
112	Int_t QueryNameKEY(MSQLServer &serv, Bool_t dummy, const char col, const char name, Bool_t insert=kTRUE)
113	{
114	TString query;
115
116	query = Form("SELECT f%sKEY FROM %s WHERE f%sName='%s'", col, col, col, name);
117	TSQLResult *res = serv.Query(query);
118	if (!res)
119	return -1;
120
121	TSQLRow *row=res->Next();
122
123	Int_t rc = row && (row)[0] ? atoi((row)[0]) : -1;
124
125	delete res;
126
127	if (rc>=0)
128	return rc;
129
130	if (!insert)
131	return -1;
132
133	//insert new value
134	query = Form("INSERT %s (f%sName) VALUES (\"%s\");", col, col, name);
135
136	if (dummy)
137	{
138	cout << query << endl;
139	return 0;
140	}
141
142	res=serv.Query(query);
143	if (!res)
144	return -1;
145
146	delete res;
147
148	Int_t key = QueryNameKEY(serv, dummy, col, name, kFALSE);
149	if (key>0)
150	{
151	cout << "New " << col << ": " << name << endl;
152	return key;
153	}
154
155	cout << "ERROR: " << query << endl;
156	return kFALSE;
157	}
158
159
160	Int_t insert(MSQLServer &serv, Bool_t dummy, TString filename)
161	{
162	ifstream fin(filename);
163	if (!fin)
164	{
165	cout << "Could not open file " << filename << endl;
166	return -1;
167	}
168
169	TString strng;
170	strng.ReadLine(fin);
171	if (strng!=TString("[CC Plain Run Summary File]"))
172	{
173	cout << filename << ": No Plain Run Summary File" << endl;
174	cout << "First Line: " << strng << endl;
175	cout << endl;
176	return -1;
177	}
178
179	strng.ReadLine(fin);
180	TRegexp reg("[0-9][0-9][0-9][0-9][0-9][0-9]-[0-9]");
181	TString arehucas = strng(reg);
182	arehucas.Prepend("20");
183	arehucas.ReplaceAll("-", "");
184
185	Int_t version = atoi(arehucas.Data());
186	if (version!=200405050 && version!=200405140 && version!=200405180 &&
187	version!=200407270 && version!=200411130 && version!=200412090 &&
188	version!=200412210 &&
189	version!=200502240 && version!=200503170 && version!=200503220 &&
190	version!=200504010 && version!=200504130 && version!=200504150 &&
191	version!=200507140 && version!=200507190 && version!=200508290 &&
192	version!=200510250 &&
193	version!=200603300 && version!=200604010 && version!=200608080 &&
194	version!=200704160 &&
195	version!=200802200)
196	{
197	cout << filename << ": File Version unknown - please update the macro!" << endl;
198	cout << "Second Line: " << strng << endl;
199	cout << endl;
200	return -1;
201	}
202
203	cout << " V" << version << " " << flush;
204
205	Int_t cnt=0;
206	while (1)
207	{
208	// ========== Col 1: Run Number =========
209	//Reading the line
210	//and converting some strings to ints/floats
211	strng.ReadToDelim(fin, ' ');
212	if (!fin)
213	break;
214
215	Int_t runnumber = atoi(strng.Data());
216
217	//runnumber=0 means no valid dataset
218	//-> continue
219	if (runnumber == 0)
220	{
221	strng.ReadLine(fin);
222	cout << "Runnumber == 0" << endl;
223	continue;
224	}
225
226	//cout << "RunNo: " << runnumber << " ";
227
228	if (ExistStr(serv, "fRunNumber", "RunData", strng.Data()))
229	{
230	// FIXME: Maybe we can implement an switch to update mode?
231	cout << "Run #" << runnumber << " already existing... skipped." << endl;
232	strng.ReadLine(fin);
233	continue;
234	}
235
236	// ========== Col 2: Run Type =========
237	strng.ReadToDelim(fin, ' ');
238	if (strng.Contains("???"))
239	strng="n/a";
240
241	Int_t runtype = QueryNameKEY(serv, dummy, "RunType", strng.Data(), kFALSE);
242	if (runtype<0)
243	{
244	cout << "ERROR - RunType " << strng << " not available." << endl;
245	strng.ReadLine(fin);
246	continue;
247	}
248
249	//cout << runtype << " ";
250
251	// ========== Col 3,4: Start Time =========
252	TString startdate, starttime;
253	startdate.ReadToDelim(fin, ' ');
254	starttime.ReadToDelim(fin, ' ');
255	//cout << startdate << " " << starttime << " ";
256
257	// ========== Col 5,6: Stop Time =========
258	TString stopdate, stoptime;
259	stopdate.ReadToDelim(fin, ' ');
260	stoptime.ReadToDelim(fin, ' ');
261	//cout << stopdate << " " << stoptime << " ";
262
263	if (startdate.Contains("???"))
264	startdate="0000-00-00";
265	if (starttime.Contains("???"))
266	starttime="00:00:00";
267	if (stopdate.Contains("???"))
268	stopdate="0000-00-00";
269	if (stoptime.Contains("???"))
270	stoptime="00:00:00";
271
272	// ========== Col 7: Source Name =========
273	strng.ReadToDelim(fin, ' ');
274	if (strng.Contains("???"))
275	strng="Unavailable";
276
277	Int_t sourcekey = QueryNameKEY(serv, dummy, "Source", strng.Data());
278	if (sourcekey<0)
279	{
280	strng.ReadLine(fin);
281	continue;
282	}
283	//cout << sourcekey << " ";
284
285	// ========== Col 8,9: Local source position =========
286	strng.ReadToDelim(fin, ' ');
287	Float_t zd = atof(strng.Data());
288
289	strng.ReadToDelim(fin, ' ');
290	Float_t az = atof(strng.Data());
291
292	//cout << zd << " " << az << " ";
293
294	// ========== Col 10: Number of Events =========
295	strng.ReadToDelim(fin, ' ');
296	Int_t evtno = atoi(strng.Data());
297
298	//cout << evtno << " ";
299
300	// ========== Col 11: Project Name =========
301	strng.ReadToDelim(fin, ' ');
302	if (strng.Contains("???"))
303	strng="Unavailable";
304
305	Int_t projkey = QueryNameKEY(serv, dummy, "Project", strng.Data());
306	if (projkey<0)
307	{
308	strng.ReadLine(fin);
309	continue;
310	}
311	//cout << projkey << " ";
312
313	// ========== Col 12: Trigger Table Name =========
314	// starting from version 200411130: Col 12,13: Trigger Table Name =========
315	strng.ReadToDelim(fin, ' ');
316	if (strng.Contains("???"))
317	strng="n/a";
318
319	Int_t l1triggerkey=1;
320	Int_t l2triggerkey=1;
321	if (version >=200411130)
322	{
323	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", strng.Data());
324	if (l1triggerkey<0)
325	{
326	strng.ReadLine(fin);
327	continue;
328	}
329
330	strng.ReadToDelim(fin, ' ');
331	if (strng.Contains("???"))
332	strng="n/a";
333
334	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", strng.Data());
335	if (l2triggerkey<0)
336	{
337	strng.ReadLine(fin);
338	continue;
339	}
340	}
341	else
342	{
343	Int_t c=0;
344
345	if (strng.Contains(":"))
346	c=1;
347
348	if (strng.Contains("L1_") && !(strng.Contains(":")))
349	c=2;
350
351	if (strng.Contains("n/a"))
352	c=3;
353
354	switch (c)
355	{
356	case 0:
357	{
358	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", strng.Data());
359	if (l2triggerkey<0)
360	{
361	strng.ReadLine(fin);
362	continue;
363	}
364
365	strng="n/a";
366	l1triggerkey = 1;
367
368	break;
369	}
370	case 1:
371	{
372	TString L1TT, L2TT;
373	L2TT=strng(7,12);
374	L1TT=strng(0,6);
375
376	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", L1TT.Data());
377	if (l1triggerkey<0)
378	{
379	strng.ReadLine(fin);
380	continue;
381	}
382
383	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", L2TT.Data());
384	if (l2triggerkey<0)
385	{
386	strng.ReadLine(fin);
387	continue;
388	}
389
390	break;
391	}
392	case 2:
393	{
394	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", strng.Data());
395	if (l1triggerkey<0)
396	{
397	strng.ReadLine(fin);
398	continue;
399	}
400
401	strng="n/a";
402	l2triggerkey = 1;
403
404	break;
405	}
406	case 3:
407	{
408	l1triggerkey = 1;
409	l2triggerkey = 1;
410	break;
411	}
412	default:
413	{
414	cout << "WARNING: neither L1 nor L2 Trigger table - please check what is happening." << strng << endl;
415	break;
416	}
417	}
418	}
419
420	// ========== Col 13-15: TrigRate, L2 UnPresc Rate, L2 Presc Rate ==========
421	strng.ReadToDelim(fin, ' ');
422	Float_t trigrate = atof(strng.Data());
423
424	strng.ReadToDelim(fin, ' ');
425	Float_t l2uprate = atof(strng.Data());
426
427	strng.ReadToDelim(fin, ' ');
428	Float_t l2prrate = atof(strng.Data());
429
430	// ========== Col 16,17: DaqRate, Storage Rate ==========
431	strng.ReadToDelim(fin, ' ');
432	Float_t daqrate = atof(strng.Data());
433
434	strng.ReadToDelim(fin, ' ');
435	Float_t storerate = atof(strng.Data());
436
437	// ========== Col 18: HV table =========
438	if (version==200405050 \|\| version==200405140)
439	strng.ReadToDelim(fin, '\n');
440	else
441	strng.ReadToDelim(fin, ' ');
442	if (strng.Contains("???"))
443	strng="n/a";
444
445	Int_t hvkey = QueryNameKEY(serv, dummy, "HvSettings", strng.Data());
446	if (hvkey<0)
447	{
448	//strng.ReadLine(fin);
449	continue;
450	}
451
452	if (version==200405180 \|\| version==200407270)
453	strng.ReadLine(fin);
454
455	Int_t testflagkey=1;
456	Int_t lightcondkey=1;
457	Int_t dttablekey=1;
458	Int_t triggerdelaytablekey=1;
459	Int_t calibrationscriptkey=1;
460	if (version==200411130 \|\| version==200412090 \|\| version==200412210
461	\|\| version==200502240 \|\| version==200503170 \|\| version==200503220
462	\|\| version==200504010 \|\| version==200504130 \|\| version==200504150
463	\|\| version==200507140 \|\| version==200507190 \|\| version==200508290
464	\|\| version==200510250 \|\| version==200603300 \|\| version==200604010
465	\|\| version==200608080 \|\| version==200704160 \|\| version==200802200)
466	{
467	// ========== Col 19-35: DC and HV-values, mjd =========
468	for (int i=0 ; i<17 ; i++)
469	{
470	strng.ReadToDelim(fin, ' ');
471	}
472
473	// ========== Col 36: test-flag =========
474	strng.ReadToDelim(fin, ' ');
475	if (strng.Contains("???"))
476	strng="n/a";
477
478	testflagkey = QueryNameKEY(serv, dummy, "TestFlag", strng.Data());
479	if (testflagkey<0)
480	{
481	strng.ReadLine(fin);
482	continue;
483	}
484
485	// ========== Col 37: light conditions =========
486	strng.ReadToDelim(fin, ' ');
487	if (strng.Contains("???"))
488	strng="n/a";
489
490	lightcondkey = QueryNameKEY(serv, dummy, "LightConditions", strng.Data());
491	if (lightcondkey<0)
492	{
493	strng.ReadLine(fin);
494	continue;
495	}
496
497	// ========== Col 38: discriminator threshold table =========
498	strng.ReadToDelim(fin, ' ');
499	if (strng.Contains("???"))
500	strng="n/a";
501
502	dttablekey = QueryNameKEY(serv, dummy, "DiscriminatorThresholdTable", strng.Data());
503	if (dttablekey<0)
504	{
505	strng.ReadLine(fin);
506	continue;
507	}
508
509	// ========== Col 39: trigger delay table =========
510	strng.ReadToDelim(fin, ' ');
511	if (strng.Contains("???"))
512	strng="n/a";
513
514	triggerdelaytablekey = QueryNameKEY(serv, dummy, "TriggerDelayTable", strng.Data());
515	if (triggerdelaytablekey<0)
516	{
517	strng.ReadLine(fin);
518	continue;
519	}
520
521	// ========== Col 40,41: Telescope RA and Dec sent to drive =========
522	strng.ReadToDelim(fin, ' ');
523	strng.ReadToDelim(fin, ' ');
524
525	// ========== Col 42: Calibration Script =========
526	if (version==200411130 \|\| version==200412090 \|\| version==200412210
527	\|\| version==200502240 \|\| version==200503170 \|\| version==200503220
528	\|\| version==200504010 \|\| version==200504130 \|\| version==200504150
529	\|\| version==200507140 \|\| version==200507190 \|\| version==200508290
530	\|\| version==200510250)
531	strng.ReadToDelim(fin, '\n');
532	else
533	strng.ReadToDelim(fin, ' ');
534	if (strng.Contains("???"))
535	strng="n/a";
536
537	calibrationscriptkey = QueryNameKEY(serv, dummy, "CalibrationScript", strng.Data());
538	if (calibrationscriptkey<0)
539	{
540	strng.ReadLine(fin);
541	continue;
542	}
543
544	}
545
546	Int_t observationmodekey=1;
547	if (version==200603300 \|\| version==200604010 \|\| version==200608080 \|\| version==200704160 \|\| version==200802200)
548	{
549	// ========== Col 43: Observation Mode =========
550	strng.ReadToDelim(fin, ' ');
551	if (strng.Contains("???"))
552	strng="n/a";
553
554	observationmodekey = QueryNameKEY(serv, dummy, "ObservationMode", strng.Data());
555	if (observationmodekey<0)
556	{
557	strng.ReadLine(fin);
558	continue;
559	}
560
561	// ========== Col 44-51: Source RA and Dec, DT's and IPR =========
562	for (int i=0 ; i<7 ; i++)
563	{
564	strng.ReadToDelim(fin, ' ');
565	}
566	strng.ReadToDelim(fin, '\n');
567	}
568
569
570	// ================================================================
571	// ========== Data read from file now access the database =========
572	// ================================================================
573
574	//assemlbe the query that is needed to insert the values of this run
575	TString query;
576	query += "INSERT RunData SET ";
577
578	query += Form("fRunNumber=%d, ", runnumber);
579	query += Form("fRunTypeKEY=%d, ", runtype);
580	query += Form("fProjectKEY=%d, ", projkey);
581	query += Form("fSourceKEY=%d, ", sourcekey);
582	query += Form("fNumEvents=%d, ", evtno);
583	query += Form("fRunStart=\"%s %s\", ", startdate.Data(), starttime.Data());
584	query += Form("fRunStop=\"%s %s\", ", stopdate.Data(), stoptime.Data());
585	query += Form("fL1TriggerTableKEY=%d, ", l1triggerkey);
586	query += Form("fL2TriggerTableKEY=%d, ", l2triggerkey);
587	query += Form("fTestFlagKEY=%d, ", testflagkey);
588	query += Form("fCalibrationScriptKEY=%d, ", calibrationscriptkey);
589	query += Form("fTriggerDelayTableKEY=%d, ", triggerdelaytablekey);
590	query += Form("fDiscriminatorThresholdTableKEY=%d, ", dttablekey);
591	query += Form("fLightConditionsKEY=%d, ", lightcondkey);
592	query += Form("fHvSettingsKEY=%d, ", hvkey);
593	query += Form("fObservationModeKEY=%d, ", observationmodekey);
594	if (!TMath::IsNaN(zd) && TMath::Finite(zd))
595	query += Form("fZenithDistance=%d, ", TMath::Nint(zd));
596	if (!TMath::IsNaN(az) && TMath::Finite(az))
597	query += Form("fAzimuth=%d, ", TMath::Nint(az));
598	if (!TMath::IsNaN(storerate) && TMath::Finite(storerate))
599	query += Form("fDaqStoreRate=%d, ", TMath::Nint(storerate));
600	if (!TMath::IsNaN(daqrate) && TMath::Finite(daqrate))
601	query += Form("fDaqTriggerRate=%d, ", TMath::Nint(daqrate));
602	if (!TMath::IsNaN(trigrate) && TMath::Finite(trigrate))
603	query += Form("fMeanTriggerRate=%d, ", TMath::Nint(trigrate));
604	if (!TMath::IsNaN(l2prrate) && TMath::Finite(l2prrate))
605	query += Form("fL2RatePresc=%d, ", TMath::Nint(l2prrate));
606	if (!TMath::IsNaN(l2uprate) && TMath::Finite(l2uprate))
607	query += Form("fL2RateUnpresc=%d, ", TMath::Nint(l2uprate));
608	query += "fMagicNumberKEY=1, fExcludedFDAKEY=1";
609
610	cnt++;
611
612	if (dummy)
613	continue;
614
615	//send query, add dataset to DB
616	TSQLResult *res = serv.Query(query);
617	if (!res)
618	return -1;
619	delete res;
620
621	//create entry in table RunProcessStatus for this runnumber
622	TString query2=Form("INSERT RunProcessStatus SET fRunNumber=%d, fTimingCorrection='1970-01-01 00:00:00', fCompmux='1970-01-01 00:00:00'",
623	runnumber);
624	if (testflagkey==3)
625	query2+=" , fDataCheckDone='1970-01-01 00:00:00'";
626	res = serv.Query(query2);
627	if (!res)
628	return -1;
629	delete res;
630	}
631
632	return cnt;
633
634	}
635
636	// This tool will work from Period017 (2004_05_17) on...
637	int filldotrun(const TString path="/data/MAGIC/Period018/ccdata", Bool_t dummy=kTRUE)
638	{
639	TEnv env("sql.rc");
640
641	MSQLServer serv(env);
642	if (!serv.IsConnected())
643	{
644	cout << "ERROR - Connection to database failed." << endl;
645	return 0;
646	}
647	cout << "filldotrun" << endl;
648	cout << "----------" << endl;
649	cout << endl;
650	cout << "Connected to " << serv.GetName() << endl;
651	cout << "Search Path: " << path << endl;
652	cout << endl;
653
654	if (path.EndsWith(".run"))
655	{
656	cout << path(TRegexp("CC_.*.run", kFALSE)) << flush;
657	Int_t n = insert(serv, dummy, path);
658	cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
659
660	return n<0 ? 2 : 1;
661	}
662
663	MDirIter Next(path, "CC_*.run", -1);
664	while (1)
665	{
666	TString name = Next();
667	if (name.IsNull())
668	break;
669
670	cout << name(TRegexp("CC_.*.run", kFALSE)) << flush;
671	Int_t n = insert(serv, dummy, name);
672	cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
673
674	if (n<0)
675	return 2;
676	}
677
678	return 1;
679	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: