source: trunk/DataCheck/QuickLook/FlareAlerts.sh@ 18766

Last change on this file since 18766 was 18743, checked in by Daniela Dorner, 8 years ago
fixed bug (for all source 3 sigma limit was used), some cleanup
  • Property svn:executable set to *
File size: 17.5 KB
Line 
1#!/bin/bash
2#
3
4# missing: install cron for current night
5# change logging to processlog
6# evaluate output of all nights
7# implement DB-entries for being called
8
9source `dirname $0`/../Sourcefile.sh
10printprocesslog "INFO starting $0"
11
12emailfrom=dorner@astro.uni-wuerzburg.de
13emailto=dorner@astro.uni-wuerzburg.de
14
15# get date
16if [ "$1" != "" ]
17then
18 checkstring=`echo $1 | grep -E -o '^20[0-9][0-9][01][0-9][0-3][0-9]$'`
19 echo $checkstring
20 if [ "$checkstring" = "" ]
21 then
22 night=`date +%Y%m%d --date="-12 HOUR"`
23 else
24 night=$1
25 fi
26else
27 night=`date +%Y%m%d --date="-12 HOUR"`
28fi
29
30echo "Processing "$night
31
32
33# get sources for current night from DB (RunInfo)
34query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunTypeKey=1 AND NOT ISNULL(fSourceKey) GROUP BY fSourceKey"
35sourcekeys=( `sendquery` )
36
37# some stuff for queries:
38ontime="IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn)"
39threshold="IF (ISNULL(fThresholdMinSet), fThresholdMedian, fThresholdMinSet)"
40cu="20.0"
41corr="1"
42# missing: corrected excessrates
43# missing: time-dependent CU
44
45function get_query_nightly_binning()
46{
47 # query to get information from DB
48 query="SELECT fSourceKey AS num, "
49 query=$query"fNight AS night, MIN(fRunID) AS runmin, MAX(fRunID) AS runmax, "
50 query=$query"MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
51 query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime, "
52 query=$query"SUM(fNumSigEvts) AS sig, SUM(fNumBgEvts) AS bg, "
53 query=$query"ROUND(SUM(fNumBgEvts)/SUM("$ontime")*3600,1) AS bgrate, "
54 query=$query"SUM(fNumExcEvts) AS exc, "
55 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts)), 1) AS excerr, "
56 query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) AS excrate, "
57 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excrateerr, "
58 query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600,1) as corexcrate, " # put here correction factor
59 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr"), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor, put here correction factor
60 query=$query"ROUND(LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)),1) AS signif, "
61 query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600/"$cu",1) AS cu, " # make value time dependent
62 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600/"$cu", 1) AS cuerr, " # make value time dependent
63 query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600/"$cu",1) as corcu, " # make value time dependent # put here correction factor
64 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr")/"$cu", 1) AS corcuerr, " # correctionfactor = exc / exc_cor # make value time dependent # put here correction factor
65 query=$query"MIN(fZenithDistanceMin) as zdmin, MAX(fZenithDistanceMax) as zdmax, "
66 query=$query"MIN("$threshold") as thmin, MAX("$threshold") as thmax "
67 query=$query"FROM AnalysisResultsRunLP "
68 query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
69 query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
70 query=$query"GROUP BY fNight, fSourceKey "
71# query=$query"ORDER BY fRunStart "
72 query=$query"HAVING ontime > 0.5 " # at least 30 minutes of observation
73}
74
75function get_query_minute_binning()
76{
77 # set binning
78 if [ "$1" != "" ]
79 then
80 bin2=$1
81 else
82 bin2=$bin
83 fi
84 # query to get information from DB
85 query="SELECT MAX(o.b) AS num, "
86 query=$query"MIN(o.n) AS night, MIN(o.run) AS runmin, MAX(o.run) AS runmax, "
87 query=$query"MIN(o.start) AS start, MAX(o.stop) AS stop, "
88 query=$query"ROUND(SUM(o.ot)/60.,1) AS ontime, "
89 query=$query"SUM(o.sig) AS sig, SUM(o.bg) AS bg, "
90 query=$query"ROUND(SUM(o.bg)/SUM(o.ot)*3600,1) AS bgrate, "
91 query=$query"SUM(o.exc) AS exc, "
92 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg)), 1) AS excerr, "
93 query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600,1) AS excrate, "
94 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600, 1) AS excrateerr, "
95 query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600,1) as corexcrate, "
96 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor
97 query=$query"ROUND(LiMa(SUM(o.sig), SUM(o.bg)),1) AS signif, "
98 query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600/o.cu,1) AS cu, "
99 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600/o.cu, 1) AS cuerr, "
100 query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600/o.cu,1) as corcu, "
101 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor)/o.cu, 1) AS corcuerr, " # correctionfactor = exc / exc_cor
102 query=$query"MIN(o.zdmin) as zdmin, MAX(o.zdmax) as zdmax, "
103 query=$query"MIN(o.th) as thmin, MAX(o.th) as thmax "
104 query=$query"FROM ("
105 query=$query"SELECT "
106 query=$query"fRunID AS run, fNight AS n, "
107 query=$query"@ot:= "$ontime" AS ot, "
108 query=$query"fRunStart AS start, fRunStop AS stop, "
109 query=$query"fNumExcEvts AS exc, fNumBgEvts AS bg, fNumSigEvts AS sig, "
110 query=$query"fNumExcEvts*"$corr" AS exccor, " # put here correction factor
111 query=$query$cu" as cu, " # make value time dependent
112 query=$query"fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "
113 query=$query$threshold" AS th, "
114 query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @bl, @bl := @bl + 1) AS b, "
115 query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @"
116 query=$query"night :=fNight AS night FROM AnalysisResultsRunLP "
117 query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
118 query=$query"CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
119 query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
120 if [ "$1" != "" ]
121 then
122 query=$query" AND fRunID <="${results[$num+3]}
123 fi
124 query=$query" ORDER BY fRunStart "
125 if [ "$1" != "" ]
126 then
127 query=$query"DESC"
128 fi
129 query=$query" ) o GROUP BY b HAVING ontime > "$bin2"*0.75 ORDER BY start "
130 if [ "$1" != "" ]
131 then
132 query=$query"DESC"
133 fi
134 #echo $query
135}
136
137function evaluate_result()
138{
139 oldexc=0
140 exc=0
141 excold=0
142 slope=0
143 slopeprev=0
144 i=0
145 # be careful with start and stop (space inbetween) -> 27 columns instead of 25
146 while [ 0 -lt 1 ]
147 do
148 trigger=0
149 num=`echo "$i * 27" | bc -l`
150 if [ "${results[$num]}" = "" ]
151 then
152 break
153 fi
154 #night=${results[$num+1]}
155 runid=${results[$num+2]}
156 sig=${results[$num+18]} # significance
157 #exc=${results[$num+14]} # excrate
158 excold=$exc
159 exc=${results[$num+19]} # excrate in CU
160
161 if [ "$onlyifhigher" = "yes" ]
162 then
163 higher=` echo " $exc > $oldexc " | bc `
164 if [ $higher -eq 1 ]
165 then
166 # keep old value
167 oldexc=$exc
168 fi
169 fi
170
171 if [ "$bin" = "" ]
172 then
173 echo " ontime: "${results[$num+8]}" h"
174 fi
175
176 # fast rise/decay trigger
177 if [ $triggertype -eq 3 ]
178 then
179 slopeprev=$slope
180 sigprev=$sig
181 if [ "$excold" = "0" ]
182 then
183 slope=0
184 else
185 slope=`echo " scale=1; ( $exc - $excold ) / ( $bin / 60. ) " | bc -l `
186 fi
187 get_query_minute_binning 60
188 #echo $query
189 results2=( `sendquery` )
190 # need to check last hour backward
191 if [ "${results2[19]}" = "" ]
192 then
193 slope60=0
194 sig60=0
195 else
196 sig60=${results2[18]} # significance 1h
197 if [ "${results2[27+19]}" = "" ]
198 then
199 # maybe treat this case differently
200 slope60=${results2[19]}
201 else
202 slope60=`echo " ${results2[19]} - ${results2[27+19]} " | bc -l` # ie /1h
203 fi
204 fi
205 #echo "exc "$exc" excold "$excold
206 #echo "slope "$slope" prev "$slopeprev" sig "$sig" prevsig "$sigprev" sig60 "$sig60" slope60 "$slope60
207 fi
208 # missing: probably one should check also 20 min binning (s example 20170103)
209 # missing: check on still available observation time
210
211 echo " "$i" "${results[$num+2]}"-"${results[$num+3]}"["${results[$num+8]}"] "$exc" "$sig
212
213 case $triggertype in
214 1) #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
215 trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
216 ;;
217 2) #echo "magic 501: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
218 trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
219 ;;
220 3) #echo "magic fast rise/decay"
221 trigger=`echo " $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit && $sig >= $siglimit " | bc -l`
222 ;;
223 *) echo $triggertype" not yet implemented"
224 ;;
225 esac
226
227 # missing: maybe use $donetriggerfile for all cases - update of nightly case can be triggered differently
228 # missing: adapt triggerfilename for type 3 (slopelimit)
229 if [ $trigger -eq 1 ]
230 then
231 # do whatever to be done to trigger
232 # - send email/sms / call
233 # - create amon file
234 # - prepare email for alert
235 # - entry in DB
236
237 if [ "$bin" = "" ]
238 then
239 # nightly file: simply overwrite, but do not send trigger again
240 # or check value if it's increasing?
241 # missing: get information from previous trigger + interpret
242 triggerfile="/home/fact/flare.alerts/"$night"-"$exclimit"_"$siglimit"-source"$sourcekey".trigger"$triggertype
243 else
244 # smaller binning: check if trigger is identical
245 triggerfile="/home/fact/flare.alerts/"$night"_"`printf %03d $runid`"-"$exclimit"_"$siglimit"-source"$sourcekey".trigger"$triggertype
246 donetriggerfile=$triggerfile".done"
247 #ls $donetriggerfile
248 # write new file only if old files do not agree
249 if [ "$donetriggerfile" != "" ] && [ -e $donetriggerfile ]
250 then
251 diff $donetriggerfile $triggerfile >/dev/null
252 checkstatus=`echo $?`
253 if [ $checkstatus -eq 0 ]
254 then
255 echo " alert already done "$donetriggerfile
256 i=`echo $i +1 | bc -l`
257 continue
258 fi
259 fi
260 if [ -e $triggerfile ]
261 then
262 mv $triggerfile $donetriggerfile
263 fi
264 fi
265 #echo $night"_"$runid" "$sourcekey" -> "$triggerfile
266 echo " writing "$triggerfile
267 touch $triggerfile
268 echo "Trigger found: " > $triggerfile
269 echo "-------------- " >> $triggerfile
270 echo " type: "$triggertype >> $triggerfile
271 echo " excess limit: "$exclimit" evts/h" >> $triggerfile
272 echo " significance limit: "$siglimit" sigma" >> $triggerfile
273 if [ "$bin" = "" ]
274 then
275 echo " nightly binning " >> $triggerfile
276 else
277 echo " binning: "$bin" min" >> $triggerfile
278 fi
279 echo "Summary of flare event: " >> $triggerfile
280 echo "----------------------- " >> $triggerfile
281 echo " source: "$sourcename >> $triggerfile
282 echo " night: "${results[$num+1]} >> $triggerfile
283 echo " runs: "${results[$num+2]}" - "${results[$num+3]} >> $triggerfile
284 echo " start: "${results[$num+4]}" "${results[$num+5]}" UTC" >> $triggerfile
285 echo " stop: "${results[$num+6]}" "${results[$num+7]}" UTC" >> $triggerfile
286 if [ "$bin" = "" ]
287 then
288 echo " ontime: "${results[$num+8]}" h" >> $triggerfile
289 else
290 echo " ontime: "${results[$num+8]}" min" >> $triggerfile
291 fi
292 #echo " ontime: "`echo "scale=1; ${results[$num+8]} / 60. " | bc -l`" min" #scale doesn't round properly
293 echo " signal: "${results[$num+9]}" evts" >> $triggerfile
294 echo " background: "${results[$num+10]}" evts" >> $triggerfile
295 echo " bgrate: "${results[$num+11]}" evts/h" >> $triggerfile
296 echo " exc: "${results[$num+12]}" +- "${results[$num+13]}" evts" >> $triggerfile
297 echo " excrate: "${results[$num+14]}" +- "${results[$num+15]}" evts/h" >> $triggerfile
298 echo " corr. excrate: "${results[$num+16]}" - "${results[$num+17]}" evts/h" >> $triggerfile
299 echo " significance: "${results[$num+18]}" sigma" >> $triggerfile
300 echo " cu: "${results[$num+19]}" +- "${results[$num+20]}" CU" >> $triggerfile
301 echo " corr. cu: "${results[$num+21]}" +- "${results[$num+22]}" CU" >> $triggerfile
302 echo " zd: "${results[$num+23]}" - "${results[$num+24]}" degree" >> $triggerfile
303 echo " th: "${results[$num+25]}" - "${results[$num+26]}" DAC counts" >> $triggerfile
304 # additional information fast rise/decay trigger
305 if [ $triggertype -eq 3 ]
306 then
307 echo "Flux doubling/halfing times: " >> $triggerfile
308 echo "---------------------------- " >> $triggerfile
309 echo " excess: "$exc >> $triggerfile
310 echo " excess old: "$excold >> $triggerfile
311 echo " significance: "$sig >> $triggerfile
312 echo " significance old: "$sigprev >> $triggerfile
313 echo " slope: "$excold >> $triggerfile
314 echo " slope old: "$slopeold >> $triggerfile
315 echo " excess 60 min: "${results2[19]} >> $triggerfile
316 echo " excess 60 min old: "${results2[27+19]} >> $triggerfile
317 echo " slope 60 min: "$slope60 >> $triggerfile
318 echo " significance 60 min: "$sig60 >> $triggerfile
319 fi
320
321 if [ "$donetriggerfile" != "" ] && [ -e $donetriggerfile ]
322 then
323 diff $donetriggerfile $triggerfile >/dev/null
324 checkstatus=`echo $?`
325 if [ $checkstatus -gt 0 ]
326 then
327 rm $donetriggerfile
328 fi
329 fi
330 # missing: get summary of whole observation
331
332 # missing AMON cases - create VOEvent-File
333
334 # send email only of $donetriggerfile doesn't exists
335 ##cat $triggerfile | mail -s 'test flare alert ' -b $emailfrom -r $emailfrom $emailto
336 #cat $triggerfile | mail -s "test flare alert for $sourcename " $emailto
337 # that's also the cases for making a call
338
339 fi
340
341 # counter
342 i=`echo $i +1 | bc -l`
343 done
344
345 echo " found "$i" data point(s)."
346 echo ""
347}
348
349for sourcekey in ${sourcekeys[@]}
350do
351 query="SELECT fSourceName FROM Source WHERE fSourceKey="$sourcekey
352 sourcename=`sendquery`
353
354 if [ $sourcekey -ne 1 ]
355 then
356 continue
357 fi
358 echo "Evaluation for $sourcename ... "
359
360 # missing: get limits from DB (structure needs to be defined)
361
362 # triggers in the frame of the MoU in the gamma-ray community
363 triggertype=1
364 # limits
365 siglimit=3.0 # sigma
366 if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ]
367 then
368 exclimit=3.0 # CU
369 else
370 exclimit=0.5 # CU
371 fi
372 # only if rate goes even higher, we have to react
373 onlyifhigher="yes"
374 higher=1
375
376 echo "[General gamma-ray MoU]"
377 echo " nightly binning..."
378 # checking nightly binning
379 bin=
380 get_query_nightly_binning
381 results=( `sendquery` )
382 evaluate_result
383
384 # 20 min binning
385 bin=20
386 echo " "$bin" min binning..."
387 get_query_minute_binning
388 results=( `sendquery` )
389 evaluate_result
390
391 # triggers to MAGIC
392 # Mrk 501 proposal
393 triggertype=2
394 if [ $sourcekey -eq 2 ]
395 then
396 siglimit=3.0
397 exclimit=2.0 # cu
398 echo "[Trigger to MAGIC 501 proposal]"
399 echo " nightly binning..."
400 # checking nightly binning
401 bin=
402 get_query_nightly_binning
403 results=( `sendquery` )
404 evaluate_result
405 # 20 min binning
406 bin=20
407 echo " "$bin" min binning..."
408 get_query_minute_binning
409 results=( `sendquery` )
410 evaluate_result
411 fi
412 # Mother of ToO - fast rise/decay
413 # sources: Mrk 421, Mrk 501, 2344, 1959
414 triggertype=3
415 if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ]
416 then
417 echo "[Trigger to MAGIC - fast rise/decay]"
418 ## keep thresholds low (or do not use in evaluation)
419 #siglimit=2.0
420 #exclimit=0.5
421 # limits in slope
422 slopelimit=1.0 # 1CU/h
423 siglimit=3.0 # 1 sigma in 1 hour
424 # binning
425 bin=30
426 echo " "$bin" min binning..."
427 get_query_minute_binning
428 results=( `sendquery` )
429 evaluate_result
430 fi
431
432 # Triggers to AMON
433 triggertype=4
434 echo "[Trigger to AMON]"
435 echo " still to be defined"
436 # missing: trigger limits and binning still to be defined
437 # also nightly?
438 # sub-threshold-triggers?
439 # FP-rate?
440 siglimit=2.0
441 exclimit=0.5
442 onlyifhigher="no"
443 bin=20
444
445 echo ""
446 echo ""
447 done
448
449finish
450
451# missing information: mjd, obs-summary, weather info (clouds? dust?)
452# prepare directly template for email
453
454# missing: error emails in case no DB content / no QLA
455# calculate delay of QLA and send email if > 30 Min
456
457# for archival testing:
458for (( i=0; i < 100 ; i++))
459do
460 date=`date --date="-${i}days" +%Y%m%d`
461 /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date
462done
463
464
Note: See TracBrowser for help on using the repository browser.