Context Navigation

source: tags/Mars-V1.1/datacenter/macros/filldotrun.C

Visit:

Last change on this file was 8424, checked in by Daniela Dorner, 18 years ago
* empty log message *
File size: 21.5 KB

Line
1	/* ======================================================================== *\
2	!
3	! *
4	! * This file is part of MARS, the MAGIC Analysis and Reconstruction
5	! * Software. It is distributed to you in the hope that it can be a useful
6	! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
7	! * It is distributed WITHOUT ANY WARRANTY.
8	! *
9	! * Permission to use, copy, modify and distribute this software and its
10	! * documentation for any purpose is hereby granted without fee,
11	! * provided that the above copyright notice appear in all copies and
12	! * that both that copyright notice and this permission notice appear
13	! * in supporting documentation. It is provided "as is" without express
14	! * or implied warranty.
15	! *
16	!
17	!
18	! Author(s): Daniela Dorner, 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
19	! Author(s): Thomas Bretz, 08/2004 <mailto:tbretz@astro.uni-wuerzburg.de>
20	!
21	! Copyright: MAGIC Software Development, 2000-2006
22	!
23	!
24	\* ======================================================================== */
25
26	/////////////////////////////////////////////////////////////////////////////
27	//
28	// filldotrun.C
29	// ============
30	//
31	// This macro is used in the datacenter to automatically fill the run-database
32	// with the information stored in the .run-files written by the central
33	// control.
34	//
35	// To following Arehucas versions are Currently supported:
36	// 040505-0, 040514-0,
37	// 040518-0, 040727-0,
38	// 041113-0, 041209-0, 041221-0
39	// 050224-0, 050317-0, 050322-0, 050401-0, 050413-0, 050415-0, 050714-0,
40	// 050719-0, 050829-0, 051025-0,
41	// 060330-0, 060401-0, 060808-0
42	// 070416-0
43	//
44	// Usage:
45	// .x filldotrun.C+("/data/MAGIC/Period019/ccdata", kTRUE)
46	//
47	// While the first argument is the directory in which all subdirectories where
48	// searches for CC_*.run files. All these files were analysed and the run
49	// info will be put into the DB, eg:
50	// "/magic/subsystemdata/cc" would do it for all data
51	// "/magic/subsystemdata/cc/2005" for one year
52	// "/magic/subsystemdata/cc/2005/11" for one month
53	// "/magic/subsystemdata/cc/2005/11/11" for a single day
54	// "/magic/subsystemdata/cc/2005/11/11/file.run" for a single file
55	//
56	// The second argument is the 'dummy-mode'. If it is kTRUE dummy-mode is
57	// switched on and nothing will be written into the database. Instead
58	// informations about the subtables are displayed. This is usefull for tests
59	// when adding a new arehucas version support. If it is kFALSE the information
60	// are written into the subtables and the runs info is written into the
61	// rundatabase.
62	//
63	// In the automatic case it makes sense to check the logfiles to make sure
64	// that everything is fine...
65	//
66	// Make sure, that database and password are corretly set in a resource
67	// file called sql.rc and the resource file is found.
68	//
69	// Remark: Running it from the commandline looks like this:
70	// root -q -l -b filldotrun.C+\(\"path\"\,kFALSE\) 2>&1 \| tee filldotrun.log
71	//
72	// Returns 0 in case of failure and 1 in case of success.
73	//
74	/////////////////////////////////////////////////////////////////////////////
75	#include <iostream>
76	#include <iomanip>
77	#include <fstream>
78
79	#include <TEnv.h>
80	#include <TMath.h>
81	#include <TRegexp.h>
82
83	#include <TSQLRow.h>
84	#include <TSQLResult.h>
85
86	#include "MTime.h"
87	#include "MDirIter.h"
88	#include "MSQLServer.h"
89
90	using namespace std;
91
92	Bool_t ExistStr(MSQLServer &serv, const char column, const char table, const char *test)
93	{
94	TString query(Form("SELECT %s FROM %s WHERE %s='%s'", column, table, column, test));
95	TSQLResult *res = serv.Query(query);
96	if (!res)
97	return kFALSE;
98
99	Bool_t rc = kFALSE;
100
101	TSQLRow *row=res->Next();
102	if (row && (*row)[0])
103	rc=kTRUE;
104
105	delete res;
106	return rc;
107	}
108
109	//get key for the value
110	// if value is not existing, insert value and get key
111	Int_t QueryNameKEY(MSQLServer &serv, Bool_t dummy, const char col, const char name, Bool_t insert=kTRUE)
112	{
113	TString query;
114
115	query = Form("SELECT f%sKEY FROM %s WHERE f%sName='%s'", col, col, col, name);
116	TSQLResult *res = serv.Query(query);
117	if (!res)
118	return -1;
119
120	TSQLRow *row=res->Next();
121
122	Int_t rc = row && (row)[0] ? atoi((row)[0]) : -1;
123
124	delete res;
125
126	if (rc>=0)
127	return rc;
128
129	if (!insert)
130	return -1;
131
132	//insert new value
133	query = Form("INSERT %s (f%sName) VALUES (\"%s\");", col, col, name);
134
135	if (dummy)
136	{
137	cout << query << endl;
138	return 0;
139	}
140
141	res=serv.Query(query);
142	if (!res)
143	return -1;
144
145	delete res;
146
147	Int_t key = QueryNameKEY(serv, dummy, col, name, kFALSE);
148	if (key>0)
149	{
150	cout << "New " << col << ": " << name << endl;
151	return key;
152	}
153
154	cout << "ERROR: " << query << endl;
155	return kFALSE;
156	}
157
158
159	Int_t insert(MSQLServer &serv, Bool_t dummy, TString filename)
160	{
161	ifstream fin(filename);
162	if (!fin)
163	{
164	cout << "Could not open file " << filename << endl;
165	return -1;
166	}
167
168	TString strng;
169	strng.ReadLine(fin);
170	if (strng!=TString("[CC Plain Run Summary File]"))
171	{
172	cout << filename << ": No Plain Run Summary File" << endl;
173	cout << "First Line: " << strng << endl;
174	cout << endl;
175	return -1;
176	}
177
178	strng.ReadLine(fin);
179	TRegexp reg("[0-9][0-9][0-9][0-9][0-9][0-9]-[0-9]");
180	TString arehucas = strng(reg);
181	arehucas.Prepend("20");
182	arehucas.ReplaceAll("-", "");
183
184	Int_t version = atoi(arehucas.Data());
185	if (version!=200405050 && version!=200405140 && version!=200405180 &&
186	version!=200407270 && version!=200411130 && version!=200412090 &&
187	version!=200412210 &&
188	version!=200502240 && version!=200503170 && version!=200503220 &&
189	version!=200504010 && version!=200504130 && version!=200504150 &&
190	version!=200507140 && version!=200507190 && version!=200508290 &&
191	version!=200510250 &&
192	version!=200603300 && version!=200604010 && version!=200608080 &&
193	version!=200704160)
194	{
195	cout << filename << ": File Version unknown - please update the macro!" << endl;
196	cout << "Second Line: " << strng << endl;
197	cout << endl;
198	return -1;
199	}
200
201	cout << " V" << version << " " << flush;
202
203	Int_t cnt=0;
204	while (1)
205	{
206	// ========== Col 1: Run Number =========
207	//Reading the line
208	//and converting some strings to ints/floats
209	strng.ReadToDelim(fin, ' ');
210	if (!fin)
211	break;
212
213	Int_t runnumber = atoi(strng.Data());
214
215	//runnumber=0 means no valid dataset
216	//-> continue
217	if (runnumber == 0)
218	{
219	strng.ReadLine(fin);
220	cout << "Runnumber == 0" << endl;
221	continue;
222	}
223
224	//cout << "RunNo: " << runnumber << " ";
225
226	if (ExistStr(serv, "fRunNumber", "RunData", strng.Data()))
227	{
228	// FIXME: Maybe we can implement an switch to update mode?
229	cout << "Run #" << runnumber << " already existing... skipped." << endl;
230	strng.ReadLine(fin);
231	continue;
232	}
233
234	// ========== Col 2: Run Type =========
235	strng.ReadToDelim(fin, ' ');
236	if (strng.Contains("???"))
237	strng="n/a";
238
239	Int_t runtype = QueryNameKEY(serv, dummy, "RunType", strng.Data(), kFALSE);
240	if (runtype<0)
241	{
242	cout << "ERROR - RunType " << strng << " not available." << endl;
243	strng.ReadLine(fin);
244	continue;
245	}
246
247	//cout << runtype << " ";
248
249	// ========== Col 3,4: Start Time =========
250	TString startdate, starttime;
251	startdate.ReadToDelim(fin, ' ');
252	starttime.ReadToDelim(fin, ' ');
253	//cout << startdate << " " << starttime << " ";
254
255	// ========== Col 5,6: Stop Time =========
256	TString stopdate, stoptime;
257	stopdate.ReadToDelim(fin, ' ');
258	stoptime.ReadToDelim(fin, ' ');
259	//cout << stopdate << " " << stoptime << " ";
260
261	if (startdate.Contains("???"))
262	startdate="0000-00-00";
263	if (starttime.Contains("???"))
264	starttime="00:00:00";
265	if (stopdate.Contains("???"))
266	stopdate="0000-00-00";
267	if (stoptime.Contains("???"))
268	stoptime="00:00:00";
269
270	// ========== Col 7: Source Name =========
271	strng.ReadToDelim(fin, ' ');
272	if (strng.Contains("???"))
273	strng="Unavailable";
274
275	Int_t sourcekey = QueryNameKEY(serv, dummy, "Source", strng.Data());
276	if (sourcekey<0)
277	{
278	strng.ReadLine(fin);
279	continue;
280	}
281	//cout << sourcekey << " ";
282
283	// ========== Col 8,9: Local source position =========
284	strng.ReadToDelim(fin, ' ');
285	Float_t zd = atof(strng.Data());
286
287	strng.ReadToDelim(fin, ' ');
288	Float_t az = atof(strng.Data());
289
290	//cout << zd << " " << az << " ";
291
292	// ========== Col 10: Number of Events =========
293	strng.ReadToDelim(fin, ' ');
294	Int_t evtno = atoi(strng.Data());
295
296	//cout << evtno << " ";
297
298	// ========== Col 11: Project Name =========
299	strng.ReadToDelim(fin, ' ');
300	if (strng.Contains("???"))
301	strng="Unavailable";
302
303	Int_t projkey = QueryNameKEY(serv, dummy, "Project", strng.Data());
304	if (projkey<0)
305	{
306	strng.ReadLine(fin);
307	continue;
308	}
309	//cout << projkey << " ";
310
311	// ========== Col 12: Trigger Table Name =========
312	// starting from version 200411130: Col 12,13: Trigger Table Name =========
313	strng.ReadToDelim(fin, ' ');
314	if (strng.Contains("???"))
315	strng="n/a";
316
317	Int_t l1triggerkey=1;
318	Int_t l2triggerkey=1;
319	if (version >=200411130)
320	{
321	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", strng.Data());
322	if (l1triggerkey<0)
323	{
324	strng.ReadLine(fin);
325	continue;
326	}
327
328	strng.ReadToDelim(fin, ' ');
329	if (strng.Contains("???"))
330	strng="n/a";
331
332	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", strng.Data());
333	if (l2triggerkey<0)
334	{
335	strng.ReadLine(fin);
336	continue;
337	}
338	}
339	else
340	{
341	Int_t c=0;
342
343	if (strng.Contains(":"))
344	c=1;
345
346	if (strng.Contains("L1_") && !(strng.Contains(":")))
347	c=2;
348
349	if (strng.Contains("n/a"))
350	c=3;
351
352	switch (c)
353	{
354	case 0:
355	{
356	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", strng.Data());
357	if (l2triggerkey<0)
358	{
359	strng.ReadLine(fin);
360	continue;
361	}
362
363	strng="n/a";
364	l1triggerkey = 1;
365
366	break;
367	}
368	case 1:
369	{
370	TString L1TT, L2TT;
371	L2TT=strng(7,12);
372	L1TT=strng(0,6);
373
374	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", L1TT.Data());
375	if (l1triggerkey<0)
376	{
377	strng.ReadLine(fin);
378	continue;
379	}
380
381	l2triggerkey = QueryNameKEY(serv, dummy, "L2TriggerTable", L2TT.Data());
382	if (l2triggerkey<0)
383	{
384	strng.ReadLine(fin);
385	continue;
386	}
387
388	break;
389	}
390	case 2:
391	{
392	l1triggerkey = QueryNameKEY(serv, dummy, "L1TriggerTable", strng.Data());
393	if (l1triggerkey<0)
394	{
395	strng.ReadLine(fin);
396	continue;
397	}
398
399	strng="n/a";
400	l2triggerkey = 1;
401
402	break;
403	}
404	case 3:
405	{
406	l1triggerkey = 1;
407	l2triggerkey = 1;
408	break;
409	}
410	default:
411	{
412	cout << "WARNING: neither L1 nor L2 Trigger table - please check what is happening." << strng << endl;
413	break;
414	}
415	}
416	}
417
418	// ========== Col 13-15: TrigRate, L2 UnPresc Rate, L2 Presc Rate ==========
419	strng.ReadToDelim(fin, ' ');
420	Float_t trigrate = atof(strng.Data());
421
422	strng.ReadToDelim(fin, ' ');
423	Float_t l2uprate = atof(strng.Data());
424
425	strng.ReadToDelim(fin, ' ');
426	Float_t l2prrate = atof(strng.Data());
427
428	// ========== Col 16,17: DaqRate, Storage Rate ==========
429	strng.ReadToDelim(fin, ' ');
430	Float_t daqrate = atof(strng.Data());
431
432	strng.ReadToDelim(fin, ' ');
433	Float_t storerate = atof(strng.Data());
434
435	// ========== Col 18: HV table =========
436	if (version==200405050 \|\| version==200405140)
437	strng.ReadToDelim(fin, '\n');
438	else
439	strng.ReadToDelim(fin, ' ');
440	if (strng.Contains("???"))
441	strng="n/a";
442
443	Int_t hvkey = QueryNameKEY(serv, dummy, "HvSettings", strng.Data());
444	if (hvkey<0)
445	{
446	//strng.ReadLine(fin);
447	continue;
448	}
449
450	if (version==200405180 \|\| version==200407270)
451	strng.ReadLine(fin);
452
453	Int_t testflagkey=1;
454	Int_t lightcondkey=1;
455	Int_t dttablekey=1;
456	Int_t triggerdelaytablekey=1;
457	Int_t calibrationscriptkey=1;
458	if (version==200411130 \|\| version==200412090 \|\| version==200412210
459	\|\| version==200502240 \|\| version==200503170 \|\| version==200503220
460	\|\| version==200504010 \|\| version==200504130 \|\| version==200504150
461	\|\| version==200507140 \|\| version==200507190 \|\| version==200508290
462	\|\| version==200510250 \|\| version==200603300 \|\| version==200604010
463	\|\| version==200608080 \|\| version==200704160)
464	{
465	// ========== Col 19-35: DC and HV-values, mjd =========
466	for (int i=0 ; i<17 ; i++)
467	{
468	strng.ReadToDelim(fin, ' ');
469	}
470
471	// ========== Col 36: test-flag =========
472	strng.ReadToDelim(fin, ' ');
473	if (strng.Contains("???"))
474	strng="n/a";
475
476	testflagkey = QueryNameKEY(serv, dummy, "TestFlag", strng.Data());
477	if (testflagkey<0)
478	{
479	strng.ReadLine(fin);
480	continue;
481	}
482
483	// ========== Col 37: light conditions =========
484	strng.ReadToDelim(fin, ' ');
485	if (strng.Contains("???"))
486	strng="n/a";
487
488	lightcondkey = QueryNameKEY(serv, dummy, "LightConditions", strng.Data());
489	if (lightcondkey<0)
490	{
491	strng.ReadLine(fin);
492	continue;
493	}
494
495	// ========== Col 38: discriminator threshold table =========
496	strng.ReadToDelim(fin, ' ');
497	if (strng.Contains("???"))
498	strng="n/a";
499
500	dttablekey = QueryNameKEY(serv, dummy, "DiscriminatorThresholdTable", strng.Data());
501	if (dttablekey<0)
502	{
503	strng.ReadLine(fin);
504	continue;
505	}
506
507	// ========== Col 39: trigger delay table =========
508	strng.ReadToDelim(fin, ' ');
509	if (strng.Contains("???"))
510	strng="n/a";
511
512	triggerdelaytablekey = QueryNameKEY(serv, dummy, "TriggerDelayTable", strng.Data());
513	if (triggerdelaytablekey<0)
514	{
515	strng.ReadLine(fin);
516	continue;
517	}
518
519	// ========== Col 40,41: Telescope RA and Dec sent to drive =========
520	strng.ReadToDelim(fin, ' ');
521	strng.ReadToDelim(fin, ' ');
522
523	// ========== Col 42: Calibration Script =========
524	if (version==200411130 \|\| version==200412090 \|\| version==200412210
525	\|\| version==200502240 \|\| version==200503170 \|\| version==200503220
526	\|\| version==200504010 \|\| version==200504130 \|\| version==200504150
527	\|\| version==200507140 \|\| version==200507190 \|\| version==200508290
528	\|\| version==200510250)
529	strng.ReadToDelim(fin, '\n');
530	else
531	strng.ReadToDelim(fin, ' ');
532	if (strng.Contains("???"))
533	strng="n/a";
534
535	calibrationscriptkey = QueryNameKEY(serv, dummy, "CalibrationScript", strng.Data());
536	if (calibrationscriptkey<0)
537	{
538	strng.ReadLine(fin);
539	continue;
540	}
541
542	}
543
544	Int_t observationmodekey=1;
545	if (version==200603300 \|\| version==200604010 \|\| version==200608080 \|\| version==200704160)
546	{
547	// ========== Col 43: Observation Mode =========
548	strng.ReadToDelim(fin, ' ');
549	if (strng.Contains("???"))
550	strng="n/a";
551
552	observationmodekey = QueryNameKEY(serv, dummy, "ObservationMode", strng.Data());
553	if (observationmodekey<0)
554	{
555	strng.ReadLine(fin);
556	continue;
557	}
558
559	// ========== Col 44-51: Source RA and Dec, DT's and IPR =========
560	for (int i=0 ; i<7 ; i++)
561	{
562	strng.ReadToDelim(fin, ' ');
563	}
564	strng.ReadToDelim(fin, '\n');
565	}
566
567
568	// ================================================================
569	// ========== Data read from file now access the database =========
570	// ================================================================
571
572	//assemlbe the query that is needed to insert the values of this run
573	TString query;
574	query += "INSERT RunData SET ";
575
576	query += Form("fRunNumber=%d, ", runnumber);
577	query += Form("fRunTypeKEY=%d, ", runtype);
578	query += Form("fProjectKEY=%d, ", projkey);
579	query += Form("fSourceKEY=%d, ", sourcekey);
580	query += Form("fNumEvents=%d, ", evtno);
581	query += Form("fRunStart=\"%s %s\", ", startdate.Data(), starttime.Data());
582	query += Form("fRunStop=\"%s %s\", ", stopdate.Data(), stoptime.Data());
583	query += Form("fL1TriggerTableKEY=%d, ", l1triggerkey);
584	query += Form("fL2TriggerTableKEY=%d, ", l2triggerkey);
585	query += Form("fTestFlagKEY=%d, ", testflagkey);
586	query += Form("fCalibrationScriptKEY=%d, ", calibrationscriptkey);
587	query += Form("fTriggerDelayTableKEY=%d, ", triggerdelaytablekey);
588	query += Form("fDiscriminatorThresholdTableKEY=%d, ", dttablekey);
589	query += Form("fLightConditionsKEY=%d, ", lightcondkey);
590	query += Form("fHvSettingsKEY=%d, ", hvkey);
591	query += Form("fObservationModeKEY=%d, ", observationmodekey);
592	if (!TMath::IsNaN(zd) && TMath::Finite(zd))
593	query += Form("fZenithDistance=%d, ", TMath::Nint(zd));
594	if (!TMath::IsNaN(az) && TMath::Finite(az))
595	query += Form("fAzimuth=%d, ", TMath::Nint(az));
596	if (!TMath::IsNaN(storerate) && TMath::Finite(storerate))
597	query += Form("fDaqStoreRate=%d, ", TMath::Nint(storerate));
598	if (!TMath::IsNaN(daqrate) && TMath::Finite(daqrate))
599	query += Form("fDaqTriggerRate=%d, ", TMath::Nint(daqrate));
600	if (!TMath::IsNaN(trigrate) && TMath::Finite(trigrate))
601	query += Form("fMeanTriggerRate=%d, ", TMath::Nint(trigrate));
602	if (!TMath::IsNaN(l2prrate) && TMath::Finite(l2prrate))
603	query += Form("fL2RatePresc=%d, ", TMath::Nint(l2prrate));
604	if (!TMath::IsNaN(l2uprate) && TMath::Finite(l2uprate))
605	query += Form("fL2RateUnpresc=%d, ", TMath::Nint(l2uprate));
606	query += "fMagicNumberKEY=1, fExcludedFDAKEY=1";
607
608	cnt++;
609
610	if (dummy)
611	continue;
612
613	//send query, add dataset to DB
614	TSQLResult *res = serv.Query(query);
615	if (!res)
616	return -1;
617	delete res;
618
619	//create entry in table RunProcessStatus for this runnumber
620	TString query2=Form("INSERT RunProcessStatus SET fRunNumber=%d, fTimingCorrection='1970-01-01 00:00:00'",
621	runnumber);
622	if (testflagkey==3)
623	query2+=" , fDataCheckDone='1970-01-01 00:00:00'";
624	res = serv.Query(query2);
625	if (!res)
626	return -1;
627	delete res;
628	}
629
630	return cnt;
631
632	}
633
634	// This tool will work from Period017 (2004_05_17) on...
635	int filldotrun(const TString path="/data/MAGIC/Period018/ccdata", Bool_t dummy=kTRUE)
636	{
637	TEnv env("sql.rc");
638
639	MSQLServer serv(env);
640	if (!serv.IsConnected())
641	{
642	cout << "ERROR - Connection to database failed." << endl;
643	return 0;
644	}
645	cout << "filldotrun" << endl;
646	cout << "----------" << endl;
647	cout << endl;
648	cout << "Connected to " << serv.GetName() << endl;
649	cout << "Search Path: " << path << endl;
650	cout << endl;
651
652	if (path.EndsWith(".run"))
653	{
654	cout << path(TRegexp("CC_.*.run", kFALSE)) << flush;
655	Int_t n = insert(serv, dummy, path);
656	cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
657
658	return n<0 ? 2 : 1;
659	}
660
661	MDirIter Next(path, "CC_*.run", -1);
662	while (1)
663	{
664	TString name = Next();
665	if (name.IsNull())
666	break;
667
668	cout << name(TRegexp("CC_.*.run", kFALSE)) << flush;
669	Int_t n = insert(serv, dummy, name);
670	cout << " <" << n << "> " << (dummy?"DUMMY":"") << endl;
671
672	if (n<0)
673	return 2;
674	}
675
676	return 1;
677	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: