source: trunk/DataCheck/Tools/get_data.sh@ 18895

Last change on this file since 18895 was 18895, checked in by Daniela Dorner, 7 years ago
added (tools to get data/fluxes from DB)
  • Property svn:executable set to *
File size: 17.2 KB
Line 
1#!/bin/bash
2
3# add:
4# - fkt f dch value?
5# - function for correction factor
6# - x-day binning
7# < 20121212 data
8# zd, th for internal
9
10# todo: check cu-factor for <20121212)
11
12# comparison:
13# function with select/table: 18.4 sec
14# function with if: 5.7 sec
15# function with if:
16# vorteil: schneller
17# nachteil: keine history
18# nachteil: es braucht 2 fkt (qla, isdc)
19
20# create function CU(night int, ana tinyint) returns double(4,1) deterministic begin declare cu double; set cu=0; select fCU into cu from CU where fValid=1 and fAnalysis=ana and fNight<night order by fNight desc limit 0,1; return cu; end $$
21
22# floor((mjd(timestamp)-mjd(startnight)+0.5)/numdays)
23# floor((mjd(timestamp)-mjd(startnight)-0.5)/numdays)
24
25
26function get_results()
27{
28 # some basic query parts
29
30 # DataCheck (old)
31 ##data with old feedback and/or different bias voltage
32 #query=$query" AND fNight>20120420 AND NOT fNight IN (20120406,20120410,20120503) AND"
33 ## broken bias channel
34 #query=$query" NOT fNight BETWEEN 20121206 AND 20130110"
35 # bg-rate cut
36 zdparam=" pow(0.753833*cos(Radians(fZenithDistanceMean)), 7.647435)*exp(-5.753686*pow(Radians(fZenithDistanceMean),2.089609))"
37 thparam=" pow((if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)-329.4203),2)*(-0.0000002044803) "
38 param=" (fNumEvtsAfterBgCuts/5-fNumSigEvts)/fOnTimeAfterCuts - "$zdparam" - "$thparam" "
39 dchold=" -0.085 < ("$param") "
40 dchold=$dchold" AND 0.25 > ("$param") "
41 # Datacheck (new) -> combine
42 dchval=" fNumEvtsAfterBgCuts/(1.41*POW(fZenithDistanceMean*PI()/180,2)+0.975)/(-7.53e-12*POW(10, LOG10(fThresholdMinSet)*3.69)+1.035)/TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))/fEffectiveOn "
43 # some semi-automatic datacheck
44 dch=" AND (("$dchval" BETWEEN 0.8 AND 1.7 AND fNight BETWEEN 20140520 AND 20150131) " #A
45 dch=$dch" OR ("$dchval" BETWEEN 0.4 AND 1.6 AND fNight BETWEEN 20150201 AND 20150715) " #B
46 dch=$dch" OR ("$dchval" BETWEEN 0.7 AND 1.4 AND fNight BETWEEN 20150716 AND 20160218) " #C
47 dch=$dch" OR ("$dchval" BETWEEN 0.5 AND 1.0 AND fNight > 20160220) " #D
48 dch=$dch" OR ("$dchold" AND fNight<20140520)) " #old
49
50 ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
51 ontime2=" fOnTimeAfterCuts "
52 ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
53 from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
54 # time range and source
55 where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
56 # some sanity checks
57 where=$where" AND fRunTypeKey=1 "
58# where=$where" AND NOT ISNULL(fNumSigEvts) AND NOT ISNULL(fNumBgEvts) "
59# where=$where" AND NOT fRunStart='0000-00-00 00:00:00' AND NOT fRunStop='0000-00-00 00:00:00' "
60 # zd cut
61 where=$where" AND fZenithDistanceMax < "$zdmax
62 # th cut
63 where=$where" AND fThresholdMedian < "$thmax
64 where=$where" "$dch
65
66 cufactor=" Avg(25.2) "
67 crabflux="3.37e-11"
68 fluxprec=13
69 crabflux="3.37"
70 fluxprec=2
71# crabflux="2.8e-11"
72# crabflux="3.9e-11"
73 # range of crab fluxes:
74 # Dortmund: 2.8e-11
75 # HESS: 3.37e-11
76 # HAWC: 3.01e-11
77 # Wue: 3.39e-11 - 3.9e-11 (ISDC analysis)
78 # 15-20% difference
79
80 case $timeunit in
81 mjd) start=" Mjd(Min(fRunStart)) "
82 stop=" Mjd(MAX(fRunStop)) "
83 deltat=" (Mjd(MAX(fRunStop))-Mjd(Min(fRunStart)))/2 "
84 time=" Mjd(Min(fRunStart))+"$deltat
85 start2=" Mjd(MIN(o.start)) "
86 stop2=" Mjd(MAX(o.stop)) "
87 deltat2=" (Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2 "
88 time2=" Mjd(MIN(o.start))+"$deltat2
89 ;;
90 unix) start="Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')) "
91 stop="Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) "
92 deltat=" (Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')))/2 "
93 time=" Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM'))+"$deltat
94 startstop2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start, "
95 startstop2=$starstop2" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop, "
96 time2=" (Unix_timestamp(CONVERT_TZ(Max(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(Min(o.start), '+00:00', 'SYSTEM')))/2 "
97 time2=" Unix_timestamp(CONVERT_TZ(Min(o.start), '+00:00', 'SYSTEM'))+"$deltat2
98 ;;
99 *) start=" MIN(fRunStart) "
100 stop=" MAX(fRunStop) "
101 deltat=" sec_to_time(time_to_sec(timediff(MAX(fRunStop), Min(fRunStart)))/2) "
102 time=" addtime(Min(fRunStart), "$deltat") "
103 start2=" MIN(o.start) "
104 stop2=" MAX(o.stop) "
105 deltat2=" sec_to_time(time_to_sec(timediff(MAX(o.stop), Min(o.start)))/2) "
106 time2=" addtime(Min(o.start), "$deltat2") "
107 ;;
108 esac
109 ontime=" SUM("$ontimeif")/60."
110 ontime2=" SUM(o.ot)/60. "
111
112 excrate=" SUM(fNumExcEvts)/SUM("$ontimeif")*3600 "
113 excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
114 significance="LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts))"
115 numexc="Sum(fNumExcEvts)"
116 numsig="Sum(fNumSigEvts)"
117 numbg="Sum(fNumBgEvts)"
118 excrateerr=" "$excerr"/SUM("$ontimeif")*3600 "
119 # thomas correction factor
120 correvts=" fNumExcEvts*(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))/((1-0.00124/1.21*(if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)-500)*(if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)>=500))) "
121 correxcrate=" SUM("$correvts")/SUM("$ontimeif")*3600 "
122 # corerr = MMath::ErrorExc(excevtssum+bgevtssum, bgevtssum*5, 0.2)/ontimesum*3600.*corrate/excrate;
123 correxcrateerr=" "$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts) "
124 # correction on run basis (not relevant for hess)
125 #cu=$correxcrate"/"$cufactor
126 cu=" SUM("$correvts"/CUQLA(fNight))/SUM("$ontimeif")*3600 "
127 #cuerr=$correxcrateerr"/"$cufactor
128 cuerr=" "$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/CUQLA(fNight))/SUM(fNumExcEvts) "
129 flux=$cu" * "$crabflux
130 fluxerr=$cuerr" * "$crabflux
131
132
133 excrate2=" (SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600 "
134 excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
135 significance2="LiMa(SUM(o.sigevts),SUM(o.bgevts))"
136 numexc2="Sum(o.sigevts-o.bgevts)"
137 numsig2="Sum(o.sigevts)"
138 numbg2="Sum(o.bgevts)"
139 excrateerr2=" "$excerr2"/SUM(o.ot)*3600 "
140 correxcrate2=" SUM(o.corevts)/SUM(o.ot)*3600 "
141 correxcrateerr2=" "$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)) "
142 #cu2=$correxcrate2"/"$cufactor
143 cu2=" SUM(o.corevts/o.cu)/SUM(o.ot)*3600 "
144 #cuerr2=$correxcrateerr2"/"$cufactor
145 cuerr2=" "$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cu)/(SUM(o.sigevts)-SUM(o.bgevts)) "
146 flux2="$cu2*"$crabflux
147 fluxerr2="$cuerr2*"$crabflux
148
149# order information such that it is easily readable by tgraph
150# tgraph: X, Y
151# tgraph errors: X, Y, EX, EY
152
153# internal
154# --------
155# timeselect:
156# mjdstar, mjdstop, mjdmean, ontime
157# excselect:
158# excrate, excerr
159# corrected: excrate, excerr
160# CU CUerr
161# flux, fluxerr
162# addselect:
163# signif
164# num exc, num sig, num bg
165# other info: zd? th?
166#
167#
168# exter nal
169# --------
170# time, delta time, start, stop
171# corr-excrate, corr-excerr
172# flux, flux-err
173
174 if [ $bin -le 0 ]
175 then
176 queryint="SELECT "
177 if [ $bin -eq 0 ]
178 then
179 queryint=$queryint" fPeriod as num, "
180 else
181 queryint=$queryint" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) as num, "
182 fi
183 queryint=$queryint" "$time" as time, "$start" as start, "$stop" as stop, "
184 queryint=$queryint" round("$excrate", 1) as excrate, round("$correxcrate", 1) as correxcrate, "
185 queryint=$queryint" round($cu, 1) as cu, $flux as flux, "
186 queryint=$queryint" "$deltat" as deltat, round("$ontime", 1) as ontime, "
187 queryint=$queryint" round("$excrateerr", 1) as excrateerr, round("$correxcrateerr", 1) as correxcrateerr, "
188 queryint=$queryint" round($cuerr, 1) as cuerr, $fluxerr as fluxerr, "
189 queryint=$queryint" round("$significance", 1) as significance, "
190 queryint=$queryint" Min(fNight) as nightmin, "
191 queryint=$queryint" Max(fNight) as nightmax, "
192 queryint=$queryint" "$numexc" as numexc, "
193 queryint=$queryint" "$numsig" as numsig, "
194 queryint=$queryint" "$numbg" as numbg "
195 #queryint=$queryint" now() as now "
196
197 queryext="SELECT "
198 if [ $bin -eq 0 ]
199 then
200 queryext=$queryext" fPeriod as num, "
201 else
202 queryext=$queryext" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) as num, "
203 fi
204 queryext=$queryext" "$time" as time, "$start" as start, "$stop" as stop, "
205 queryext=$queryext" round("$correxcrate", 1) as correxcrate, round($flux, "$fluxprec") as flux, "
206 queryext=$queryext" "$deltat" as deltat, round("$ontime", 1) as ontime, "
207 queryext=$queryext" round("$correxcrateerr", 1) as correxcrateerr, round($fluxerr, "$fluxprec") as fluxerr, "
208 queryext=$queryext" round("$significance", 1) as significance "
209 #queryext=$queryext" now() as now "
210
211 querybase=$from$where
212 #if [ $bin -eq -1 ]
213 #then
214 # querybase=$querybase" GROUP BY fNight "
215 #fi
216 #querybase=$querybase" GROUP BY FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".)"
217 querybase=$querybase" GROUP BY num "
218 #echo "-----------------"$querybase
219 if [ "$ontimelimit" = "" ]
220 then
221 querybase=$querybase" HAVING SUM("$ontimeif")>1200 AND NOT SUM(fNumBgEvts)=SUM(fNumSigEvts) ORDER BY num " # 20 min
222 else
223 querybase=$querybase" HAVING SUM("$ontimeif")>"$ontimelimit" AND NOT SUM(fNumBgEvts)=SUM(fNumSigEvts) ORDER BY num "
224 fi
225
226 queryint=$queryint" "$querybase
227 queryext=$queryext" "$querybase
228 else
229 queryint="SELECT "
230 queryint=$queryint" "$time2" as time, "$start2" as start, "$stop2" as stop, "
231 queryint=$queryint" round("$excrate2", 1) as excrate, round("$correxcrate2", 1) as correxcrate, "
232 queryint=$queryint" round("$cu2", 1) as cu, round("$flux2", "$fluxprec") as flux, "
233 queryint=$queryint" round("$excrateerr2", 1) as excrateerr, round("$correxcrateerr2", 1) as correxcrateerr, "
234 queryint=$queryint" "$deltat2" as deltat, round("$ontime2", 1) as ontime, "
235 queryint=$queryint" round("$cuerr2", 1) as cuerr, round("$fluxerr2", "$fluxprec") as fluxerr, "
236 queryint=$queryint" round("$significance2", 1) as significance, "
237 queryint=$queryint" avg(o.night) as night, "
238 queryint=$queryint" "$numexc2" as numexc, "
239 queryint=$queryint" "$numsig2" as numsig, "
240 queryint=$queryint" "$numbg2" as numbg "
241 #queryint=$queryint" now() as now "
242
243 queryext="SELECT "
244 queryext=$queryext" "$time2" as time, "$start2" as start, "$stop2" as stop, "
245 queryext=$queryext" round("$correxcrate2", 1) as correxcrate, round("$flux2", "$fluxprec") as flux, "
246 queryext=$queryext" "$deltat2" as deltat, round("$ontime2", 1) as ontime, "
247 queryext=$queryext" round("$correxcrateerr2", 1) as correxcrateerr, round("$fluxerr2", "$fluxprec") as fluxerr, "
248 queryext=$queryext" round("$significance2", 1) as significance "
249 #queryext=$queryext" now() as now "
250
251 querybase=" FROM (SELECT fNight, @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "
252 querybase=$querybase" "$correvts" AS corevts, CUQLA(fNight) AS cu, "
253 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
254 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
255 querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
256 querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING NOT sum(o.bgevts)=sum(o.sigevts) AND ontime>0.75*"$bin
257 # AND NOT SUM(fNumBgEvts)=SUM(fNumSigEvts)
258
259 queryint=$queryint" "$querybase
260 queryext=$queryext" "$querybase
261 fi
262
263
264 fileint=$path"/data/FACT_preliminary_"$name"_internal.dat"
265 if [ "$overwrite" = "yes" ]
266 then
267 echo "internal: "$fileint
268 echo "# this file was created at "`date` > $fileint
269 fi
270 if [ $bin -le 0 ]
271 then
272 echo "# numbin time[mjd] start[mjd] stop[mjd] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time[mjd] ontime[min] excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance nightmin, nightmax num_exc num_sig num_bg " >> $fileint
273 else
274 echo "# time[mjd] start[mjd] stop[mjd] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time[mjd] ontime[min] excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance nightmin, nightmax num_exc num_sig num_bg " >> $fileint
275 fi
276 #echo "$queryint"
277 mysql --defaults-file=$sqlpw -u root $dbname -s -e "$queryint" >> $fileint
278 #mysql --defaults-file=$sqlpw -u root $dbname -e "$queryint"
279
280
281 fileext=$path"/data/FACT_preliminary_"$name".dat"
282 if [ "$overwrite" = "yes" ]
283 then
284 echo "external: "$fileext
285 echo "# this file was created at "`date` > $fileext
286 fi
287 if [ $bin -lt 0 ]
288 then
289 echo "# numbin time[mjd] start[mjd] stop[mjd] corr.excrate[evts/h] flux[e-11/cm2/s] delta_time[mjd] ontime[min] corr.excrate_err[evts/h] flux_err[e-11/cm2/s] significance " >> $fileext
290 else
291 echo "# time[mjd] start[mjd] stop[mjd] corr.excrate[evts/h] flux[e-11/cm2/s] delta_time[mjd] ontime[min] corr.excrate_err[evts/h] flux_err[e-11/cm2/s] significance " >> $fileext
292 fi
293 #echo "$queryext"
294 mysql --defaults-file=$sqlpw -u root $dbname -s -e "$queryext" >> $fileext
295 #mysql --defaults-file=$sqlpw -u root $dbname -e "$queryext"
296}
297
298# setup
299# db
300sqlpw=/home/$USER/.mysql.pw
301dbname=factdata20170804
302# selection
303timeunit=mjd
304#bin=20 # min
305#bin=0 # period
306#bin=-365 # yearly
307bin=-1 # nightly
308zdmax=90
309thmax=1500
310path=`dirname $0`
311table="AnalysisResultsRunLP"
312table="AnalysisResultsAllQLA"
313overwrite="yes"
314bin=-1 # nightly
315nightmin=20111115
316nightmax=20171231
317
318
319# Mrk 421
320source=1
321name="Mrk421_nightly"
322bin=-1
323get_results
324name="Mrk421_20min"
325bin=20
326get_results
327name="Mrk421_3d"
328bin=-3
329get_results
330name="Mrk421_10d"
331bin=-10
332get_results
333name="Mrk421_period"
334bin=0
335get_results
336
337
338
339# Mrk 501
340source=2
341name="Mrk501_nightly"
342bin=-1
343get_results
344name="Mrk501_20min"
345bin=20
346get_results
347name="Mrk501_3d"
348bin=-3
349get_results
350name="Mrk501_10d"
351bin=-10
352get_results
353name="Mrk501_period"
354bin=0
355get_results
356
357
358
359# 2344
360source=3
361name="2344_nightly"
362bin=-1
363get_results
364name="2344_20min"
365bin=20
366get_results
367name="2344_period"
368bin=0
369get_results
370
371
372
373# 1959
374source=7
375name="1959_nightly"
376bin=-1
377get_results
378name="1959_20min"
379bin=20
380get_results
381name="1959_period"
382bin=0
383get_results
384
385
386
387# 0323
388source=12
389name="0323_nightly"
390bin=-1
391get_results
392name="0323_20min"
393bin=20
394get_results
395name="0323_period"
396bin=0
397get_results
398
399
400
401# crab
402source=5
403name="Crab_nightly"
404bin=-1
405get_results
406name="Crab_20min"
407bin=20
408get_results
409name="Crab_period"
410bin=0
411get_results
412name="Crab_season"
413bin=-365
414nightmin=20110716
415nightmax=20180716
416get_results
417
418
419
420exit
421
422# more examples
423
424name="1959_2016"
425source=7
426bin=-1
427nightmin=20160201
428nightmax=20161105
429get_results
430
431name="1959_all_variable"
432overwrite="no"
433source=7
434bin=-365
435nightmin=20120201
436nightmax=20130131
437get_results
438nightmin=20130201
439nightmax=20140131
440get_results
441nightmin=20140201
442nightmax=20150131
443get_results
444bin=0
445nightmin=20150201
446nightmax=20160131
447get_results
448bin=-1
449nightmin=20160201
450nightmax=20170131
451get_results
452bin=0
453nightmin=20170201
454nightmax=20180131
455get_results
456
457
458
459overwrite="yes"
460name="1959_all_variable2"
461overwrite="no"
462source=7
463bin=-365
464nightmin=20120201
465nightmax=20130131
466get_results
467nightmin=20130201
468nightmax=20140131
469get_results
470nightmin=20140201
471nightmax=20150131
472get_results
473bin=0
474nightmin=20150201
475nightmax=20160131
476get_results
477bin=-1
478nightmin=20160201
479nightmax=20160817
480get_results
481bin=0
482nightmin=20160818
483nightmax=20180131
484get_results
485
486
487
488exit
489
490overwrite="yes"
491
492bin=0
493source=3
494name="2344period"
495get_results
496
497exit
498
499# flare night (HESS)
500name="Mrk501_10min_flarenight"
501source=2
502bin=10
503nightmin=20140623
504nightmax=20140623
505get_results
506
507
508exit
509
510# flare night (HESS)
511name="Mrk501_5min_flarenight"
512source=2
513bin=5
514nightmin=20140623
515nightmax=20140623
516get_results
517
518
519exit
520
521
522# full sample
523name="Mrk421_all_nightly"
524source=1
525get_results
526
527name="Mrk501_all_nightly"
528source=2
529get_results
530
531name="1959_all_nightly"
532source=7
533get_results
534
535name="2344_all_nightly"
536source=3
537get_results
538
539exit
540
541
542name="HESE20160427"
543source=19
544nightmin=20160425
545bin=-10
546get_results
547
548name="AMON20160731"
549source=21
550nightmin=20160730
551bin=-10
552get_results
553
554
555# full sample
556name="Mrk501_all_3d"
557source=2
558bin=-3
559get_results
560
561exit
562
563# full sample
564name="Mrk421_all_3d"
565source=1
566bin=-3
567get_results
568
569name="1959_2016_nightly"
570source=7
571nightmin=20160301
572nightmax=20160831
573get_results
574
575# full sample
576name="Mrk421_all_nightly"
577source=1
578get_results
579
580name="Mrk501_all_nightly"
581source=2
582get_results
583
584name="1959_all_nightly"
585source=7
586get_results
587
588bin=20 # min
589
590# full sample
591name="Mrk421_all_20min"
592source=1
593get_results
594
595name="Mrk501_all_20min"
596source=2
597get_results
598
599name="1959_all_20min"
600source=7
601get_results
602
603exit
604
605name="Mrk501_all_weekly"
606nightmin=20140524
607nightmax=20140930
608source=2
609bin=-7
610get_results
611
612exit
613
614
Note: See TracBrowser for help on using the repository browser.