source: trunk/DataCheck/QuickLook/FlareAlerts.sh@ 18951

Last change on this file since 18951 was 18906, checked in by Daniela Dorner, 7 years ago
fixed two bugs, added filling of informaion in DB
  • Property svn:executable set to *
File size: 29.8 KB
Line 
1#!/bin/bash
2#
3
4# missing:
5# finalize DB-entries for being called
6# prepare directly template for email
7# add x-ray trigger as trigger type or
8# add note for values > 70 evts/h for X-ray triggers
9#
10# add more information:
11# mjd, obs-summary, weather info (clouds? dust?), CU, corrected excrates
12#
13# evaluate output of all nights (check also for crab to estimate fluctuations)
14# error emails in case no DB content / no QLA
15# calculate delay of QLA and send email if > 30 Min
16
17# run for more nights:
18# for (( i=0; i < 70 ; i++)); do date=`date --date="-${i}days" +%Y%m%d`; /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date; done > logfile
19
20source `dirname $0`/../Sourcefile.sh
21printprocesslog "INFO starting $0"
22
23emailfrom=dorner@astro.uni-wuerzburg.de
24emailto=fact-online@lists.phys.ethz.ch
25
26if [ ! -e $flarealertspath ] || [ "$flarealertspath" == "" ]
27then
28 echo "flarealertspath "$flarealertspath" missing on "$HOSTNAME
29 printprocesslog "ERROR flarealertspath "$flarealertspath" missing on "$HOSTNAME
30 finish
31fi
32voeventpath="/home/fact/amon/flare_alerts/"
33
34logfile=$runlogpath"/FlareAlerts-"$datetime".log"
35date > $logfile
36
37# get date
38if [ "$1" != "" ]
39then
40 checkstring=`echo $1 | grep -E -o '^20[0-9][0-9][01][0-9][0-3][0-9]$'`
41 echo $checkstring
42 if [ "$checkstring" = "" ]
43 then
44 night=`date +%Y%m%d --date="-12 HOUR"`
45 else
46 night=$1
47 fi
48else
49 night=`date +%Y%m%d --date="-12 HOUR"`
50fi
51
52echo "Processing "$night >> $logfile
53
54
55# get sources for current night from DB (RunInfo)
56query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunTypeKey=1 AND NOT ISNULL(fSourceKey) GROUP BY fSourceKey"
57sourcekeys=( `sendquery` )
58if [ ${#sourcekeys[@]} -eq 0 ]
59then
60 echo " No sources found for "$night >> $logfile
61 finish
62fi
63
64printprocesslog "INFO Checking the "${#sourcekeys[@]}" sourcekeys: "${sourcekeys[@]}
65
66# some stuff for queries:
67ontime="IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn)"
68threshold="IF (ISNULL(fThresholdMinSet), fThresholdMedian, fThresholdMinSet)"
69#cu="20.0"
70cu="CUQLA(fNight)"
71corr="1"
72# missing: corrected excessrates
73
74
75# missing: excerr is NOT rate
76function get_query_nightly_binning()
77{
78 # query to get information from DB
79 query="SELECT fSourceKey AS num, "
80 query=$query"fNight AS night, MIN(fRunID) AS runmin, MAX(fRunID) AS runmax, "
81 query=$query"MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
82 query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime, "
83 query=$query"SUM(fNumSigEvts) AS sig, SUM(fNumBgEvts) AS bg, "
84 query=$query"ROUND(SUM(fNumBgEvts)/SUM("$ontime")*3600,1) AS bgrate, "
85 query=$query"SUM(fNumExcEvts) AS exc, "
86 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts)), 1) AS excerr, "
87 query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) AS excrate, "
88 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excrateerr, "
89 query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600,1) as corexcrate, " # put here correction factor
90 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr"), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor, put here correction factor
91 query=$query"ROUND(LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)),1) AS signif, "
92 query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600/"$cu",1) AS cu, " # make value time dependent
93 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600/"$cu", 1) AS cuerr, " # make value time dependent
94 query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600/"$cu",1) as corcu, " # make value time dependent # put here correction factor
95 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr")/"$cu", 1) AS corcuerr, " # correctionfactor = exc / exc_cor # make value time dependent # put here correction factor
96 query=$query"MIN(fZenithDistanceMin) as zdmin, MAX(fZenithDistanceMax) as zdmax, "
97 query=$query"MIN("$threshold") as thmin, MAX("$threshold") as thmax "
98 query=$query"FROM AnalysisResultsRunLP "
99 query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
100 query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
101 query=$query"GROUP BY fNight, fSourceKey "
102# query=$query"ORDER BY fRunStart "
103 query=$query"HAVING ontime > 0.5 " # at least 30 minutes of observation
104}
105
106function get_query_minute_binning()
107{
108 # set binning
109 if [ "$1" != "" ]
110 then
111 bin2=$1
112 else
113 bin2=$bin
114 fi
115 # query to get information from DB
116 query="SELECT MAX(o.b) AS num, "
117 query=$query"MIN(o.n) AS night, MIN(o.run) AS runmin, MAX(o.run) AS runmax, "
118 query=$query"MIN(o.start) AS start, MAX(o.stop) AS stop, "
119 query=$query"ROUND(SUM(o.ot)/60.,1) AS ontime, "
120 query=$query"SUM(o.sig) AS sig, SUM(o.bg) AS bg, "
121 query=$query"ROUND(SUM(o.bg)/SUM(o.ot)*3600,1) AS bgrate, "
122 query=$query"SUM(o.exc) AS exc, "
123 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg)), 1) AS excerr, "
124 query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600,1) AS excrate, "
125 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600, 1) AS excrateerr, "
126 query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600,1) as corexcrate, "
127 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor
128 query=$query"ROUND(LiMa(SUM(o.sig), SUM(o.bg)),1) AS signif, "
129 query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600/o.cu,1) AS cu, "
130 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600/o.cu, 1) AS cuerr, "
131 query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600/o.cu,1) as corcu, "
132 query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor)/o.cu, 1) AS corcuerr, " # correctionfactor = exc / exc_cor
133 query=$query"MIN(o.zdmin) as zdmin, MAX(o.zdmax) as zdmax, "
134 query=$query"MIN(o.th) as thmin, MAX(o.th) as thmax "
135 query=$query"FROM ("
136 query=$query"SELECT "
137 query=$query"fRunID AS run, fNight AS n, "
138 query=$query"@ot:= "$ontime" AS ot, "
139 query=$query"fRunStart AS start, fRunStop AS stop, "
140 query=$query"fNumExcEvts AS exc, fNumBgEvts AS bg, fNumSigEvts AS sig, "
141 query=$query"fNumExcEvts*"$corr" AS exccor, " # put here correction factor
142 query=$query$cu" as cu, " # make value time dependent
143 query=$query"fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "
144 query=$query$threshold" AS th, "
145 query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @bl, @bl := @bl + 1) AS b, "
146 query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @"
147 query=$query"night :=fNight AS night FROM AnalysisResultsRunLP "
148 query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
149 query=$query"CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
150 query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
151# if [ "$1" != "" ]
152# then
153# query=$query" AND fRunID <="${results[$num+3]}
154# fi
155 query=$query" ORDER BY fRunStart "
156# if [ "$1" != "" ]
157# then
158# query=$query"DESC"
159# fi
160 query=$query" ) o GROUP BY b HAVING ontime > "$bin2"*0.75 ORDER BY start "
161# if [ "$1" != "" ]
162# then
163# query=$query"DESC"
164# fi
165 #echo $query
166}
167
168function print_voevent_file()
169{
170 # put here voevent file
171 # make sure that it is written to amon-folder
172 echo '<?xml version="1.0" ?>'
173 echo '<voe:VOEvent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
174 echo 'xmlns:voe="http://www.ivoa.net/xml/VOEvent/v2.0"'
175 echo 'xsi:schemaLocation="http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd"'
176 echo ' version="2.0" role="test" ivorn="ivo://amon/fact_#5_3772_0">'
177 echo ' <Who>'
178 echo ' <AuthorIVORN>ivo://FACT</AuthorIVORN>'
179 echo ' <Date>'`date +%F\ %H:%M:%S`'</Date>'
180 echo ' </Who>'
181 # FACT = 5 in AMON
182 echo ' <What>'
183 echo ' <Param name="stream" dataType="float" value="5" ucd="meta.number" unit=" ">'
184 echo ' <Description>Stream number</Description>'
185 echo ' </Param>'
186 # event identifier
187 # nightly binning: night+sourcekey
188 # 20 min binning: night+source+?
189 echo ' <Param name="id" dataType="float" value="'${night}`printf %03d $runid`${sourcekey}'" ucd="meta.number" unit=" ">'
190 echo ' <Description>Id number</Description>'
191 echo ' </Param>'
192 # keep 0 for the beginning
193 # how to handle updates? e.g. in case of full disk and random processing
194 echo ' <Param name="rev" dataType="float" value="0" ucd="meta.number" unit=" ">'
195 echo ' <Description>Revision number</Description>'
196 echo ' </Param>'
197 # what to do with this? in IC it's number of neutrinos, but does number of gammas make sense?
198 # (excevts? depends on time range...)
199 echo ' <Param name="nevents" dataType="float" value="1" ucd="meta.number" unit=" ">'
200 echo ' <Description>Number of events</Description>'
201 echo ' </Param>'
202 # get time window (stop of last - start of first run)
203 # maybe: don't send alert if time window > xxx
204 echo ' <Param name="deltaT" dataType="float" value="0.0" ucd="time.duration" unit="s">'
205 echo ' <Description>Time window of the burst</Description>'
206 echo ' </Param>'
207 # probably 0
208 echo ' <Param name="sigmaT" dataType="float" value="0.0" ucd="time" unit="s">'
209 echo ' <Description>Uncertainty of the time window</Description>'
210 echo ' </Param>'
211 # FPR - to be calculated
212 echo ' <Param name="false_pos" dataType="float" value="-1" ucd="stat.probability" unit="s-1.sr-1 ">'
213 echo ' <Description>False positive rate</Description>'
214 echo ' </Param>'
215 # some significane value - how significant is the alert?
216 # to be discussed and calculated
217 echo ' <Param name="pvalue" dataType="float" value="1.0" ucd="stat.probability" unit=" ">'
218 echo ' <Description>Pvalue</Description>'
219 echo ' </Param>'
220 # use source RA/Dec? but then naming doesn't fit
221 # -> leave empty for the moment
222 echo ' <Param name="point_RA" dataType="float" value="-1.0" ucd="os.eq.ra" unit="deg">'
223 echo ' <Description>Pointing RA</Description>'
224 echo ' </Param>'
225 echo ' <Param name="point_dec" dataType="float" value="-1.0" ucd="os.eq.dec" unit="deg">'
226 echo ' <Description>Pointing Dec</Description>'
227 echo ' </Param>'
228 # shape of psf - to be determined - fit thetaplot? for crab or mc
229 echo ' <Param name="psf_type" dataType="string" value="fisher" ucd="meta.code.multip" unit=" ">'
230 echo ' <Description>Type of psf (skymap, fisher, kent, king)</Description>'
231 echo ' </Param>'
232 echo ' <Group name="aux_params">'
233 # which other information to add?
234 echo ' <Param name="zenith" dataType="float" value="0.0" ucd="os.lc.ze" unit="deg"/>'
235 echo ' <Param name="xyz" dataType="float" value="0.0" ucd="os.lc.ze" unit="xyz"/>'
236 echo ' </Group>'
237 echo ' </What>'
238 echo ' <WhereWhen>'
239 echo ' <ObsDataLocation>'
240 echo ' <ObservatoryLocation>'
241 echo ' <AstroCoordSystem id="UTC-GEOD-TOPO"/>'
242 echo ' <AstroCoords coord_system_id="UTC-GEOD-TOPO">'
243 echo ' <Position3D unit="deg-deg-m">'
244 echo ' <Name1>longitude</Name1>'
245 echo ' <Name2>latitude</Name2>'
246 echo ' <Name3>elevation</Name3>'
247 echo ' <Value3>'
248 echo ' <C1>-17.88</C1>'
249 echo ' <C2>28.76</C2>'
250 echo ' <C3>2200</C3>'
251 echo ' </Value3>'
252 echo ' </Position3D>'
253 echo ' </AstroCoords>'
254 echo ' </ObservatoryLocation>'
255 echo ' <ObservationLocation>'
256 echo ' <AstroCoordSystem id="UTC-ICRS-TOPO"/>'
257 echo ' <AstroCoords coord_system_id="UTC-ICRS-TOPO">'
258 echo ' <Time unit="s">'
259 echo ' <TimeInstant>'
260 # python needs the .0 in the time-format - as I don't have the time more accurately here, I put .0
261 echo ' <ISOTime>'${results[$num+4]}' '${results[$num+5]}'.0</ISOTime>'
262 echo ' </TimeInstant>'
263 echo ' </Time>'
264 echo ' <Position2D unit="deg-deg">'
265 echo ' <Name1>RA</Name1>'
266 echo ' <Name2>Dec</Name2>'
267 echo ' <Value2>'
268 echo ' <C1>'${sourceinfo[2]}'</C1>' # RA in deg
269 echo ' <C2>'${sourceinfo[0]}'</C2>' # decl in deg
270 echo ' </Value2>'
271 echo ' <Error2Radius>0.1</Error2Radius>' # PSF of FACT
272 echo ' </Position2D>'
273 echo ' </AstroCoords>'
274 echo ' </ObservationLocation>'
275 echo ' </ObsDataLocation>'
276 echo ' </WhereWhen>'
277 echo ' <Description>FACT flare event information</Description>'
278 echo '</voe:VOEvent>'
279
280}
281
282function evaluate_result()
283{
284 oldexc=0
285 exc=0
286 excold=0
287 slope=0
288 slopeprev=0
289 i=0
290 # be careful with start and stop (space inbetween) -> 27 columns instead of 25
291 while [ 0 -lt 1 ]
292 do
293 trigger=0
294 num=`echo "$i * 27" | bc -l`
295 if [ "${results[$num]}" = "" ]
296 then
297 break
298 fi
299 #night=${results[$num+1]}
300 runid=${results[$num+2]}
301 sig=${results[$num+18]} # significance
302 #exc=${results[$num+14]} # excrate
303 excold=$exc
304 exc=${results[$num+19]} # excrate in CU
305
306 if [ "$onlyifhigher" = "yes" ]
307 then
308 higher=` echo " $exc > $oldexc " | bc `
309 if [ $higher -eq 1 ]
310 then
311 # keep old value
312 oldexc=$exc
313 fi
314 fi
315
316 if [ "$bin" = "" ]
317 then
318 echo " ontime: "${results[$num+8]}" h" >> $logfile
319 fi
320
321 # fast rise/decay trigger
322 if [ $triggertype -eq 3 ]
323 then
324 slopeprev=$slope
325 sigprev=$sig
326 if [ "$excold" = "0" ]
327 then
328 slope=0
329 else
330 slope=`echo " scale=1; ( $exc - $excold ) / ( $bin / 60. ) " | bc -l `
331 fi
332 get_query_minute_binning 60
333 #echo $query
334 results2=( `sendquery` )
335 # need to check last hour backward
336 if [ "${results2[19]}" = "" ]
337 then
338 slope60=0
339 sig60=0
340 else
341 sig60=${results2[18]} # significance 1h
342 if [ "${results2[27+19]}" = "" ]
343 then
344 # maybe treat this case differently
345 slope60=${results2[19]}
346 else
347 slope60=`echo " ${results2[19]} - ${results2[27+19]} " | bc -l` # ie /1h
348 fi
349 fi
350 #echo "exc "$exc" excold "$excold
351 #echo "slope "$slope" prev "$slopeprev" sig "$sig" prevsig "$sigprev" sig60 "$sig60" slope60 "$slope60
352 fi
353 # missing: probably one should check also 20 min binning (s example 20170103)
354 # missing: check on still available observation time
355
356 echo " "$i" "${results[$num+2]}"-"${results[$num+3]}"["${results[$num+8]}"] "$exc" "$sig >> $logfile
357
358 case $triggertype in
359 1) #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
360 trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
361 limits=$exclimit"_"$siglimit
362 ;;
363 2) #echo "magic 501: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
364 trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
365 limits=$exclimit"_"$siglimit
366 ;;
367 3) #echo "magic fast rise/decay"
368 #trigger=`echo " $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit && $sig >= $siglimit " | bc -l`
369 trigger=`echo " ( ( $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit ) || ( - $slope >= $slopelimit && - $slopeprev >= $slopelimit && - $slope60 >= $slopelimit ) ) && $sig >= $siglimit " | bc -l`
370 limits=$slopelimit"_"$siglimit
371 ;;
372 5) #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
373 trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
374 limits=$exclimit"_"$siglimit
375 ;;
376 *) echo $triggertype" not yet implemented" >> $logfile
377 ;;
378 esac
379
380 # missing: maybe use $donetriggerfile for all cases - update of nightly case can be triggered differently
381 # missing: adapt triggerfilename for type 3 (slopelimit)
382 if [ $trigger -eq 1 ]
383 then
384 # do whatever to be done to trigger
385 # - send email/sms / call
386 # - create amon file
387 # - prepare email for alert
388 # - entry in DB
389
390 if [ "$bin" = "" ]
391 then
392 # nightly file: simply overwrite, but do not send trigger again
393 # or check value if it's increasing?
394 # missing: get information from previous trigger + interpret
395 triggerfile=$flarealertspath"/"$night"-"$limits"-source"$sourcekey".trigger"$triggertype
396 else
397 # smaller binning: check if trigger is identical
398 triggerfile=$flarealertspath"/"$night"_"`printf %03d $runid`"-"$limits"-source"$sourcekey".trigger"$triggertype
399 fi
400 donetriggerfile=$triggerfile".done"
401 #ls $donetriggerfile
402
403 # check if nightly flux increased
404 if [ "$bin" = "" ]
405 then
406 trigger2=1
407 changedfiles=( `ls $donetriggerfile"-changed-"* 2>/dev/null` )
408 if [ ${#changedfiles[@]} -gt 0 ]
409 then
410 oldnightlyexc=`grep "corr. cu: [0-9].[0-9] " ${changedfiles[@]} | grep -o -E ' [0-9].[0-9] ' | sort | tail -1`
411 trigger2=`echo " $exc > $oldnightlyexc " | bc -l`
412 echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")"
413 echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")" >> $logfile
414 fi
415 fi
416
417 # write new file only if old files do not agree
418 if [ -e $donetriggerfile ]
419 then
420 diff $donetriggerfile $triggerfile >/dev/null
421 checkstatus=`echo $?`
422 if [ $checkstatus -eq 0 ]
423 then
424 echo " alert already done "$donetriggerfile >> $logfile
425 i=`echo $i +1 | bc -l`
426 continue
427 fi
428 fi
429 if [ -e $triggerfile ]
430 then
431 mv $triggerfile $donetriggerfile
432 fi
433
434 #echo $night"_"$runid" "$sourcekey" -> "$triggerfile
435 echo " writing "$triggerfile >> $logfile
436 touch $triggerfile
437 echo "Trigger found: " > $triggerfile
438 echo "-------------- " >> $triggerfile
439 echo " type: "$triggertype >> $triggerfile
440 echo " excess limit: "$exclimit" CU" >> $triggerfile
441 echo " significance limit: "$siglimit" sigma" >> $triggerfile
442 if [ "$bin" = "" ]
443 then
444 echo " nightly binning " >> $triggerfile
445 else
446 echo " binning: "$bin" min" >> $triggerfile
447 fi
448 echo "Summary of flare event: " >> $triggerfile
449 echo "----------------------- " >> $triggerfile
450 echo " source: "$sourcename >> $triggerfile
451 echo " night: "${results[$num+1]} >> $triggerfile
452 echo " runs: "${results[$num+2]}" - "${results[$num+3]} >> $triggerfile
453 echo " start: "${results[$num+4]}" "${results[$num+5]}" UTC" >> $triggerfile
454 echo " stop: "${results[$num+6]}" "${results[$num+7]}" UTC" >> $triggerfile
455 if [ "$bin" = "" ]
456 then
457 echo " ontime: "${results[$num+8]}" h" >> $triggerfile
458 else
459 echo " ontime: "${results[$num+8]}" min" >> $triggerfile
460 fi
461 #echo " ontime: "`echo "scale=1; ${results[$num+8]} / 60. " | bc -l`" min" #scale doesn't round properly
462 echo " signal: "${results[$num+9]}" evts" >> $triggerfile
463 echo " background: "${results[$num+10]}" evts" >> $triggerfile
464 echo " bgrate: "${results[$num+11]}" evts/h" >> $triggerfile
465 echo " exc: "${results[$num+12]}" +- "${results[$num+13]}" evts" >> $triggerfile
466 echo " excrate: "${results[$num+14]}" +- "${results[$num+15]}" evts/h" >> $triggerfile
467 echo " corr. excrate: "${results[$num+16]}" - "${results[$num+17]}" evts/h" >> $triggerfile
468 echo " significance: "${results[$num+18]}" sigma" >> $triggerfile
469 echo " cu: "${results[$num+19]}" +- "${results[$num+20]}" CU" >> $triggerfile
470 echo " corr. cu: "${results[$num+21]}" +- "${results[$num+22]}" CU" >> $triggerfile
471 echo " zd: "${results[$num+23]}" - "${results[$num+24]}" degree" >> $triggerfile
472 echo " th: "${results[$num+25]}" - "${results[$num+26]}" DAC counts" >> $triggerfile
473 # additional information fast rise/decay trigger
474 if [ $triggertype -eq 3 ]
475 then
476 echo "Flux doubling/halfing times: " >> $triggerfile
477 echo "---------------------------- " >> $triggerfile
478 echo " excess: "$exc >> $triggerfile
479 echo " excess old: "$excold >> $triggerfile
480 echo " significance: "$sig >> $triggerfile
481 echo " significance old: "$sigprev >> $triggerfile
482 echo " slope: "$excold >> $triggerfile
483 echo " slope old: "$slopeold >> $triggerfile
484 echo " excess 60 min: "${results2[19]} >> $triggerfile
485 echo " excess 60 min old: "${results2[27+19]} >> $triggerfile
486 echo " slope 60 min: "$slope60 >> $triggerfile
487 echo " significance 60 min: "$sig60 >> $triggerfile
488 fi
489
490 if [ -e $donetriggerfile ]
491 then
492 diff $donetriggerfile $triggerfile >/dev/null
493 checkstatus=`echo $?`
494 if [ $checkstatus -gt 0 ]
495 then
496 # keep history of non-sent triggers
497 donetriggerfile2=$donetriggerfile"-changed-"`date +%Y%m%d%H%M%S`
498 cp $donetriggerfile $donetriggerfile2
499 fi
500 fi
501 # missing: get summary of whole observation
502
503 # send email only of $donetriggerfile doesn't exists
504 if ! [ -e $donetriggerfile ]
505 then
506 query="INSERT FlareAlerts.FlareTriggers SET fTriggerInserted=Now(), fNight="$night", fRunID="$runid", fTriggerType="$triggertype
507 if [ "$bin" = "" ]
508 then
509 query=$query", fBinning=NULL"
510 else
511 query=$query", fBinning="$bin
512 fi
513 echo $query
514 sendquery >> $logfile
515
516 # AMON case: create VOEvent-File
517 if [ $triggertype -eq 5 ]
518 then
519 voeventfile=$voeventpath"/"`basename $triggerfile`".xml"
520 # missing: check for archive file
521 # if exist -> do revision
522 echo "creating "$voeventfile
523 echo "Creating "$voeventfile >> $logfile
524 #print_voevent_file
525 print_voevent_file > $voeventfile
526 # in amon-case no email needs to be sent
527 else
528 if [ "$bin" = "" ] && [ $trigger2 -eq 0 ]
529 then
530 continue
531 fi
532 echo "sending["$triggertype"] "$triggerfile
533 echo "sending["$triggertype"] "$triggerfile >> $logfile
534 cat $triggerfile | mail -s 'test flare alert ' -b $emailfrom -r $emailfrom $emailto
535 #cat $triggerfile | mail -s "test flare alert for $sourcename " $emailto
536 # that's also the cases for making a call
537 # fill DB for shifthelper
538 fi
539 fi
540 fi
541
542 # counter
543 i=`echo $i +1 | bc -l`
544 done
545
546 echo " found "$i" data point(s)." >> $logfile
547 echo "" >> $logfile
548}
549
550get_average_flux()
551{
552 # query average flux from DB
553 query="SELECT ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) as excrate, "
554 query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excerr, "
555 query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime "
556 query=$query"FROM AnalysisResultsRunLP "
557 query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
558 if [ "$1" = "" ]
559 then
560 query=$query"WHERE fSourceKey="$sourcekey" AND NOT ISNULL(fNumExcEvts) "
561 else
562 query=$query"WHERE fSourceKey="$sourcekey" AND fNight BETWEEN "$1" AND "$2" AND NOT ISNULL(fNumExcEvts) "
563 fi
564 query=$query"GROUP BY fSourceKey "
565 sendquery
566 # missing - get in CU to correct for fluctuations of CU
567}
568
569
570
571# main part of the script
572
573# missing: check if twistd client is running
574# + check if there are remaining files in the to-send-folder of amon
575# -> send email if amon connection has problem
576
577
578for sourcekey in ${sourcekeys[@]}
579do
580 query="SELECT fSourceName FROM Source WHERE fSourceKey="$sourcekey
581 sourcename=`sendquery` #do not combine this with other source info as sourcename can have spaces
582
583 # todo: what about data check ?
584 # should avg include current night?
585 total=( `get_average_flux` )
586 month=( `get_average_flux \`date -d $night' - 1 MONTH' +%Y%m%d\` $night` )
587 year=( `get_average_flux \`date -d $night' - 1 YEAR' +%Y%m%d\` $night` )
588 avgflux=${total[0]}
589 avgfluxmonth=${month[0]}
590 avgfluxyear=${year[0]}
591 error=${total[1]}
592 errormonth=${month[1]}
593 erroryear=${year[1]}
594
595 # getting some information on the source
596 query="SELECT fDeclination, fRightAscension, fRightAscension/24.*15 FROM Source WHERE fSourceKey="$sourcekey
597 sourceinfo=( `sendquery` )
598
599 # ignore Crab
600 if [ $sourcekey -eq 5 ]
601 then
602 continue
603 fi
604 printprocesslog "INFO Evaluation for $sourcename ... "
605 echo "Evaluation for $sourcename ... " >> $logfile
606 echo " average fluxes: "$avgfluxmonth"+-"$errormonth" evts/h (last month) "$avgfluxyear"+-"$erroryear" evts/h (last year) "$avgflux"+-"$error" evts/h (all)" >> $logfile
607
608 # missing: get limits from DB (structure needs to be defined)
609
610 # triggers in the frame of the MoU in the gamma-ray community
611 triggertype=1
612 # limits
613 siglimit=3.0 # sigma
614 #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ]
615 if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 5 ]
616 then
617 exclimit=3.0 # CU
618 else
619 exclimit=0.5 # CU
620 fi
621 # only if rate goes even higher, we have to react
622 onlyifhigher="yes"
623 higher=1
624
625 printprocesslog "INFO checking for [General gamma-ray MoU]" >> $logfile
626 echo "[General gamma-ray MoU]" >> $logfile
627 echo " nightly binning..." >> $logfile
628 # checking nightly binning
629 bin=
630 get_query_nightly_binning
631 results=( `sendquery` )
632 evaluate_result
633
634 # 20 min binning
635 bin=20
636 echo " "$bin" min binning..." >> $logfile
637 get_query_minute_binning $bin
638 results=( `sendquery` )
639 evaluate_result
640
641 # triggers to MAGIC
642 # Mrk 501 proposal
643 triggertype=2
644 if [ $sourcekey -eq 2 ]
645 then
646 siglimit=3.0
647 exclimit=2.0 # cu
648 printprocesslog "INFO checking for [Trigger to MAGIC 501 proposal]" >> $logfile
649 echo "[Trigger to MAGIC 501 proposal]" >> $logfile
650 # checking nightly binning
651 bin=
652 echo " nightly binning..." >> $logfile
653 get_query_nightly_binning
654 results=( `sendquery` )
655 evaluate_result
656 # 20 min binning
657 bin=20
658 echo " "$bin" min binning..." >> $logfile
659 get_query_minute_binning $bin
660 results=( `sendquery` )
661 evaluate_result
662 fi
663 # Mother of ToO - fast rise/decay
664 # sources: Mrk 421, Mrk 501, 2344, 1959
665 triggertype=3
666 #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ] || [ $sourcekey -eq 5 ] # for testing
667 if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ]
668 then
669 printprocesslog "INFO checking for [Trigger to MAGIC - fast rise/decay]" >> $logfile
670 echo "[Trigger to MAGIC - fast rise/decay]" >> $logfile
671 ## keep thresholds low (or do not use in evaluation)
672 #siglimit=2.0
673 #exclimit=0.5
674 # limits in slope
675 slopelimit=1.0 # 1CU/h
676 siglimit=3.0 # 1 sigma in 1 hour
677 # binning
678 bin=30
679 echo " "$bin" min binning..." >> $logfile
680 get_query_minute_binning $bin
681 results=( `sendquery` )
682 evaluate_result
683 fi
684
685 # X-ray ToO
686 triggertype=4
687 # to be added
688
689 # AMON - automatic triggers using VOEvent files
690 triggertype=5
691 printprocesslog "INFO checking for [Trigger to AMON]" >> $logfile
692 echo "[Trigger to AMON]" >> $logfile
693 echo " details still to be defined" >> $logfile
694 # missing: trigger limits and binning still to be defined
695 # x times above average + significance limit ?
696 # sub-threshold?
697 # FP-rate to be calculated
698 # use for the moment 0.5 CU and 3 sigma
699 siglimit=3.0
700 exclimit=0.5
701 onlyifhigher="no"
702 # checking nightly binning
703 bin=
704 echo " nightly binning..." >> $logfile
705 get_query_nightly_binning
706 results=( `sendquery` )
707 evaluate_result
708 bin=20
709 echo " "$bin" min binning..." >> $logfile
710 get_query_minute_binning $bin
711 results=( `sendquery` )
712 evaluate_result
713
714 echo "" >> $logfile
715 echo "" >> $logfile
716done
717
718finish
719
720# for archival testing:
721for (( i=0; i < 100 ; i++))
722do
723 date=`date --date="-${i}days" +%Y%m%d`
724 /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date
725done
726
727
Note: See TracBrowser for help on using the repository browser.