source: trunk/DataCheck/QuickLook/FlareAlerts.sh @ 18906

Last change on this file since 18906 was 18906, checked in by dorner, 22 months ago
fixed two bugs, added filling of informaion in DB
  • Property svn:executable set to *
File size: 29.8 KB
Line 
1#!/bin/bash
2#
3
4# missing:
5# finalize DB-entries for being called
6# prepare directly template for email
7# add x-ray trigger as trigger type or
8#    add note for values > 70 evts/h for X-ray triggers
9#
10# add more information:
11#   mjd, obs-summary, weather info (clouds? dust?), CU, corrected excrates
12#
13# evaluate output of all nights (check also for crab to estimate fluctuations)
14# error emails in case no DB content / no QLA
15# calculate delay of QLA and send email if > 30 Min
16
17# run for more nights:
18# for (( i=0; i < 70 ; i++)); do date=`date --date="-${i}days" +%Y%m%d`; /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date; done > logfile
19
20source `dirname $0`/../Sourcefile.sh
21printprocesslog "INFO starting $0"
22
23emailfrom=dorner@astro.uni-wuerzburg.de
24emailto=fact-online@lists.phys.ethz.ch
25
26if [ ! -e $flarealertspath ] || [ "$flarealertspath" == "" ]
27then 
28   echo "flarealertspath "$flarealertspath" missing on "$HOSTNAME
29   printprocesslog "ERROR flarealertspath "$flarealertspath" missing on "$HOSTNAME
30   finish
31fi
32voeventpath="/home/fact/amon/flare_alerts/"
33
34logfile=$runlogpath"/FlareAlerts-"$datetime".log"
35date > $logfile
36
37# get date
38if [ "$1" != "" ]
39then
40   checkstring=`echo $1 | grep -E -o '^20[0-9][0-9][01][0-9][0-3][0-9]$'`
41   echo $checkstring
42   if [ "$checkstring" = "" ]
43   then
44      night=`date +%Y%m%d --date="-12 HOUR"`
45   else
46      night=$1
47   fi
48else
49   night=`date +%Y%m%d --date="-12 HOUR"`
50fi
51
52echo "Processing "$night >> $logfile
53
54
55# get sources for current night from DB (RunInfo)
56query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunTypeKey=1 AND NOT ISNULL(fSourceKey) GROUP BY fSourceKey"
57sourcekeys=( `sendquery` )
58if [ ${#sourcekeys[@]} -eq 0 ]
59then
60   echo " No sources found for "$night >> $logfile
61   finish
62fi
63
64printprocesslog "INFO Checking the "${#sourcekeys[@]}" sourcekeys: "${sourcekeys[@]}
65
66# some stuff for queries:
67ontime="IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn)"
68threshold="IF (ISNULL(fThresholdMinSet), fThresholdMedian, fThresholdMinSet)"
69#cu="20.0"
70cu="CUQLA(fNight)"
71corr="1"
72# missing: corrected excessrates
73
74
75# missing: excerr is NOT rate
76function get_query_nightly_binning()
77{
78   # query to get information from DB
79   query="SELECT fSourceKey AS num, "
80   query=$query"fNight AS night, MIN(fRunID) AS runmin, MAX(fRunID) AS runmax, "
81   query=$query"MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
82   query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime, "
83   query=$query"SUM(fNumSigEvts) AS sig, SUM(fNumBgEvts) AS bg, "
84   query=$query"ROUND(SUM(fNumBgEvts)/SUM("$ontime")*3600,1) AS bgrate, "
85   query=$query"SUM(fNumExcEvts) AS exc, "
86   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts)), 1) AS excerr, "
87   query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) AS excrate, "
88   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excrateerr, "
89   query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600,1) as corexcrate, " # put here correction factor
90   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr"), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor, put here correction factor
91   query=$query"ROUND(LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)),1) AS signif, "
92   query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600/"$cu",1) AS cu, " # make value time dependent
93   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600/"$cu", 1) AS cuerr, " # make value time dependent
94   query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600/"$cu",1) as corcu, " # make value time dependent # put here correction factor
95   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr")/"$cu", 1) AS corcuerr, " # correctionfactor = exc / exc_cor # make value time dependent # put here correction factor
96   query=$query"MIN(fZenithDistanceMin) as zdmin, MAX(fZenithDistanceMax) as zdmax, "
97   query=$query"MIN("$threshold") as thmin, MAX("$threshold") as thmax "
98   query=$query"FROM AnalysisResultsRunLP "
99   query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
100   query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
101   query=$query"GROUP BY fNight, fSourceKey "
102#   query=$query"ORDER BY fRunStart "
103   query=$query"HAVING ontime > 0.5 " # at least 30 minutes of observation
104}
105
106function get_query_minute_binning()
107{
108   # set binning
109   if [ "$1" != "" ]
110   then
111      bin2=$1
112   else
113      bin2=$bin
114   fi
115   # query to get information from DB
116   query="SELECT MAX(o.b) AS num, "
117   query=$query"MIN(o.n) AS night, MIN(o.run) AS runmin, MAX(o.run) AS runmax, "
118   query=$query"MIN(o.start) AS start, MAX(o.stop) AS stop, "
119   query=$query"ROUND(SUM(o.ot)/60.,1) AS ontime, "
120   query=$query"SUM(o.sig) AS sig, SUM(o.bg) AS bg, "
121   query=$query"ROUND(SUM(o.bg)/SUM(o.ot)*3600,1) AS bgrate, "
122   query=$query"SUM(o.exc) AS exc, "
123   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg)), 1) AS excerr, "
124   query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600,1) AS excrate, "
125   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600, 1) AS excrateerr, "
126   query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600,1) as corexcrate, "
127   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor
128   query=$query"ROUND(LiMa(SUM(o.sig), SUM(o.bg)),1) AS signif, "
129   query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600/o.cu,1) AS cu, "
130   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600/o.cu, 1) AS cuerr, "
131   query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600/o.cu,1) as corcu, "
132   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor)/o.cu, 1) AS corcuerr, " # correctionfactor = exc / exc_cor
133   query=$query"MIN(o.zdmin) as zdmin, MAX(o.zdmax) as zdmax, "
134   query=$query"MIN(o.th) as thmin, MAX(o.th) as thmax "
135   query=$query"FROM ("
136   query=$query"SELECT "
137   query=$query"fRunID AS run, fNight AS n, "
138   query=$query"@ot:= "$ontime" AS ot, "
139   query=$query"fRunStart AS start, fRunStop AS stop, "
140   query=$query"fNumExcEvts AS exc, fNumBgEvts AS bg, fNumSigEvts AS sig, "
141   query=$query"fNumExcEvts*"$corr" AS exccor, " # put here correction factor
142   query=$query$cu" as cu, " # make value time dependent
143   query=$query"fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "
144   query=$query$threshold" AS th, "
145   query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @bl, @bl := @bl + 1) AS b,  "
146   query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @"
147   query=$query"night :=fNight AS night FROM AnalysisResultsRunLP "
148   query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
149   query=$query"CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
150   query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
151#   if [ "$1" != "" ]
152#   then
153#      query=$query" AND fRunID <="${results[$num+3]}
154#   fi
155   query=$query" ORDER BY fRunStart "
156#   if [ "$1" != "" ]
157#   then
158#      query=$query"DESC"
159#   fi
160   query=$query" ) o GROUP BY b HAVING ontime > "$bin2"*0.75 ORDER BY start "
161#   if [ "$1" != "" ]
162#   then
163#      query=$query"DESC"
164#   fi
165   #echo $query
166}
167
168function print_voevent_file()
169{
170   # put here voevent file
171   # make sure that it is written to amon-folder
172   echo '<?xml version="1.0" ?>'
173   echo '<voe:VOEvent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
174   echo 'xmlns:voe="http://www.ivoa.net/xml/VOEvent/v2.0"'
175   echo 'xsi:schemaLocation="http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd"'
176   echo ' version="2.0" role="test" ivorn="ivo://amon/fact_#5_3772_0">'
177   echo '    <Who>'
178   echo '        <AuthorIVORN>ivo://FACT</AuthorIVORN>'
179   echo '        <Date>'`date +%F\ %H:%M:%S`'</Date>'
180   echo '    </Who>'
181   # FACT = 5 in AMON
182   echo '    <What>'
183   echo '        <Param name="stream" dataType="float" value="5" ucd="meta.number" unit=" ">'
184   echo '            <Description>Stream number</Description>'
185   echo '        </Param>'
186   # event identifier
187   # nightly binning: night+sourcekey
188   # 20 min binning: night+source+?
189   echo '        <Param name="id" dataType="float" value="'${night}`printf %03d $runid`${sourcekey}'" ucd="meta.number" unit=" ">'
190   echo '            <Description>Id number</Description>'
191   echo '        </Param>'
192   # keep 0 for the beginning
193   # how to handle updates? e.g. in case of full disk and random processing
194   echo '        <Param name="rev" dataType="float" value="0" ucd="meta.number" unit=" ">'
195   echo '            <Description>Revision number</Description>'
196   echo '        </Param>'
197   # what to do with this? in IC it's number of neutrinos, but does number of gammas make sense?
198   #  (excevts? depends on time range...)
199   echo '        <Param name="nevents" dataType="float" value="1" ucd="meta.number" unit=" ">'
200   echo '            <Description>Number of events</Description>'
201   echo '        </Param>'
202   # get time window (stop of last - start of first run)
203   # maybe: don't send alert if time window > xxx
204   echo '        <Param name="deltaT" dataType="float" value="0.0" ucd="time.duration" unit="s">'
205   echo '            <Description>Time window of the burst</Description>'
206   echo '        </Param>'
207   # probably 0
208   echo '        <Param name="sigmaT" dataType="float" value="0.0" ucd="time" unit="s">'
209   echo '            <Description>Uncertainty of the time window</Description>'
210   echo '        </Param>'
211   # FPR - to be calculated
212   echo '        <Param name="false_pos" dataType="float" value="-1" ucd="stat.probability" unit="s-1.sr-1 ">'
213   echo '            <Description>False positive rate</Description>'
214   echo '        </Param>'
215   # some significane value - how significant is the alert?
216   # to be discussed and calculated
217   echo '        <Param name="pvalue" dataType="float" value="1.0" ucd="stat.probability" unit=" ">'
218   echo '            <Description>Pvalue</Description>'
219   echo '        </Param>'
220   # use source RA/Dec? but then naming doesn't fit
221   # -> leave empty for the moment
222   echo '        <Param name="point_RA" dataType="float" value="-1.0" ucd="os.eq.ra" unit="deg">'
223   echo '            <Description>Pointing RA</Description>'
224   echo '        </Param>'
225   echo '        <Param name="point_dec" dataType="float" value="-1.0" ucd="os.eq.dec" unit="deg">'
226   echo '            <Description>Pointing Dec</Description>'
227   echo '        </Param>'
228   # shape of psf - to be determined - fit thetaplot? for crab or mc
229   echo '        <Param name="psf_type" dataType="string" value="fisher" ucd="meta.code.multip" unit=" ">'
230   echo '            <Description>Type of psf (skymap, fisher, kent, king)</Description>'
231   echo '        </Param>'
232   echo '        <Group name="aux_params">'
233   # which other information to add?
234   echo '            <Param name="zenith" dataType="float" value="0.0" ucd="os.lc.ze" unit="deg"/>'
235   echo '            <Param name="xyz" dataType="float" value="0.0" ucd="os.lc.ze" unit="xyz"/>'
236   echo '        </Group>'
237   echo '    </What>'
238   echo '    <WhereWhen>'
239   echo '        <ObsDataLocation>'
240   echo '            <ObservatoryLocation>'
241   echo '                <AstroCoordSystem id="UTC-GEOD-TOPO"/>'
242   echo '                <AstroCoords coord_system_id="UTC-GEOD-TOPO">'
243   echo '                    <Position3D unit="deg-deg-m">'
244   echo '                        <Name1>longitude</Name1>'
245   echo '                        <Name2>latitude</Name2>'
246   echo '                        <Name3>elevation</Name3>'
247   echo '                        <Value3>'
248   echo '                            <C1>-17.88</C1>'
249   echo '                            <C2>28.76</C2>'
250   echo '                            <C3>2200</C3>'
251   echo '                        </Value3>'
252   echo '                    </Position3D>'
253   echo '                </AstroCoords>'
254   echo '            </ObservatoryLocation>'
255   echo '            <ObservationLocation>'
256   echo '                <AstroCoordSystem id="UTC-ICRS-TOPO"/>'
257   echo '                <AstroCoords coord_system_id="UTC-ICRS-TOPO">'
258   echo '                    <Time unit="s">'
259   echo '                        <TimeInstant>'
260   # python needs the .0 in the time-format - as I don't have the time more accurately here, I put .0
261   echo '                            <ISOTime>'${results[$num+4]}' '${results[$num+5]}'.0</ISOTime>'
262   echo '                        </TimeInstant>'
263   echo '                    </Time>'
264   echo '                    <Position2D unit="deg-deg">'
265   echo '                        <Name1>RA</Name1>'
266   echo '                        <Name2>Dec</Name2>'
267   echo '                        <Value2>'
268   echo '                            <C1>'${sourceinfo[2]}'</C1>' # RA in deg
269   echo '                            <C2>'${sourceinfo[0]}'</C2>' # decl in deg
270   echo '                        </Value2>'
271   echo '                        <Error2Radius>0.1</Error2Radius>' # PSF of FACT
272   echo '                    </Position2D>'
273   echo '                </AstroCoords>'
274   echo '            </ObservationLocation>'
275   echo '        </ObsDataLocation>'
276   echo '    </WhereWhen>'
277   echo '    <Description>FACT flare event information</Description>'
278   echo '</voe:VOEvent>'
279   
280}
281
282function evaluate_result()
283{
284   oldexc=0
285   exc=0
286   excold=0
287   slope=0
288   slopeprev=0
289   i=0
290   # be careful with start and stop (space inbetween) -> 27 columns instead of 25
291   while [ 0 -lt 1 ]
292   do
293      trigger=0
294      num=`echo "$i * 27" | bc -l`
295      if [ "${results[$num]}" = "" ]
296      then
297         break
298      fi
299      #night=${results[$num+1]}
300      runid=${results[$num+2]}
301      sig=${results[$num+18]} # significance
302      #exc=${results[$num+14]} # excrate
303      excold=$exc
304      exc=${results[$num+19]} # excrate in CU
305     
306      if [ "$onlyifhigher" = "yes" ]
307      then
308         higher=` echo " $exc > $oldexc " | bc `
309         if [ $higher -eq 1 ]
310         then
311            # keep old value
312            oldexc=$exc
313         fi
314      fi
315     
316      if [ "$bin" = "" ]   
317      then 
318         echo "  ontime: "${results[$num+8]}" h" >> $logfile
319      fi
320
321      # fast rise/decay trigger
322      if [ $triggertype -eq 3 ]
323      then
324         slopeprev=$slope
325         sigprev=$sig
326         if [ "$excold" = "0" ]
327         then
328            slope=0
329         else
330            slope=`echo " scale=1; ( $exc - $excold ) / ( $bin / 60. ) " | bc -l `
331         fi
332         get_query_minute_binning 60
333         #echo $query
334         results2=( `sendquery` )
335         # need to check last hour backward
336         if [ "${results2[19]}" = "" ]
337         then 
338            slope60=0
339            sig60=0
340         else
341            sig60=${results2[18]} # significance 1h
342            if [ "${results2[27+19]}" = "" ]
343            then 
344               # maybe treat this case differently
345               slope60=${results2[19]}
346            else
347               slope60=`echo " ${results2[19]} - ${results2[27+19]} " | bc -l` # ie /1h
348            fi
349         fi
350         #echo "exc "$exc" excold "$excold
351         #echo "slope "$slope" prev "$slopeprev" sig "$sig" prevsig "$sigprev" sig60 "$sig60" slope60 "$slope60
352      fi
353      # missing: probably one should check also 20 min binning (s example 20170103)
354      # missing: check on still available observation time
355
356      echo "    "$i" "${results[$num+2]}"-"${results[$num+3]}"["${results[$num+8]}"] "$exc" "$sig >> $logfile
357     
358      case $triggertype in
359         1)   #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
360              trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
361              limits=$exclimit"_"$siglimit
362              ;;
363         2)   #echo "magic 501: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
364              trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
365              limits=$exclimit"_"$siglimit
366              ;;
367         3)   #echo "magic fast rise/decay"
368              #trigger=`echo " $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit && $sig >= $siglimit " | bc -l`
369              trigger=`echo " ( ( $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit ) || ( - $slope >= $slopelimit && - $slopeprev >= $slopelimit && - $slope60 >= $slopelimit  ) ) && $sig >= $siglimit " | bc -l`
370              limits=$slopelimit"_"$siglimit
371              ;;
372         5)   #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
373              trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
374              limits=$exclimit"_"$siglimit
375              ;;
376         *)   echo $triggertype" not yet implemented" >> $logfile
377              ;;
378      esac
379
380      # missing: maybe use $donetriggerfile for all cases - update of nightly case can be triggered differently
381      # missing: adapt triggerfilename for type 3 (slopelimit)
382      if [ $trigger -eq 1 ]
383      then
384         # do whatever to be done to trigger
385         #  - send email/sms / call
386         #  - create amon file
387         #  - prepare email for alert
388         #  - entry in DB
389         
390         if [ "$bin" = "" ]
391         then 
392            # nightly file: simply overwrite, but do not send trigger again
393            # or check value if it's increasing?
394            # missing: get information from previous trigger + interpret
395            triggerfile=$flarealertspath"/"$night"-"$limits"-source"$sourcekey".trigger"$triggertype
396         else
397            # smaller binning: check if trigger is identical
398            triggerfile=$flarealertspath"/"$night"_"`printf %03d $runid`"-"$limits"-source"$sourcekey".trigger"$triggertype
399         fi
400         donetriggerfile=$triggerfile".done"
401         #ls $donetriggerfile
402
403         # check if nightly flux increased
404         if [ "$bin" = "" ]
405         then 
406            trigger2=1
407            changedfiles=( `ls $donetriggerfile"-changed-"* 2>/dev/null` )
408            if [ ${#changedfiles[@]} -gt 0 ]
409            then 
410               oldnightlyexc=`grep "corr. cu:      [0-9].[0-9] " ${changedfiles[@]} | grep -o -E ' [0-9].[0-9] ' | sort | tail -1`
411               trigger2=`echo " $exc > $oldnightlyexc " | bc -l`
412               echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")"
413               echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")" >> $logfile
414            fi
415         fi
416         
417         # write new file only if old files do not agree
418         if [ -e $donetriggerfile ]
419         then
420            diff $donetriggerfile $triggerfile >/dev/null
421            checkstatus=`echo $?`
422            if [ $checkstatus -eq 0 ]
423            then
424               echo "  alert already done "$donetriggerfile >> $logfile
425               i=`echo $i +1 | bc -l`
426               continue
427            fi
428         fi
429         if [ -e $triggerfile ]
430         then
431            mv $triggerfile $donetriggerfile
432         fi
433         
434         #echo $night"_"$runid" "$sourcekey" -> "$triggerfile
435         echo "  writing "$triggerfile >> $logfile
436         touch $triggerfile
437         echo "Trigger found: " > $triggerfile
438         echo "-------------- " >> $triggerfile
439         echo "  type: "$triggertype >> $triggerfile
440         echo "  excess limit: "$exclimit" CU" >> $triggerfile
441         echo "  significance limit: "$siglimit" sigma" >> $triggerfile
442         if [ "$bin" = "" ]   
443         then 
444            echo "  nightly binning " >> $triggerfile
445         else
446            echo "  binning: "$bin" min" >> $triggerfile
447         fi
448         echo "Summary of flare event: " >> $triggerfile
449         echo "----------------------- " >> $triggerfile
450         echo "  source:        "$sourcename >> $triggerfile
451         echo "  night:         "${results[$num+1]} >> $triggerfile
452         echo "  runs:          "${results[$num+2]}" - "${results[$num+3]} >> $triggerfile
453         echo "  start:         "${results[$num+4]}" "${results[$num+5]}" UTC" >> $triggerfile
454         echo "  stop:          "${results[$num+6]}" "${results[$num+7]}" UTC" >> $triggerfile
455         if [ "$bin" = "" ]
456         then 
457            echo "  ontime:        "${results[$num+8]}" h" >> $triggerfile
458         else
459            echo "  ontime:        "${results[$num+8]}" min" >> $triggerfile
460         fi
461         #echo "  ontime:        "`echo "scale=1; ${results[$num+8]} / 60. " | bc -l`" min" #scale doesn't round properly
462         echo "  signal:        "${results[$num+9]}" evts" >> $triggerfile
463         echo "  background:    "${results[$num+10]}" evts" >> $triggerfile
464         echo "  bgrate:        "${results[$num+11]}" evts/h" >> $triggerfile
465         echo "  exc:           "${results[$num+12]}" +- "${results[$num+13]}" evts" >> $triggerfile
466         echo "  excrate:       "${results[$num+14]}" +- "${results[$num+15]}" evts/h" >> $triggerfile
467         echo "  corr. excrate: "${results[$num+16]}" - "${results[$num+17]}" evts/h" >> $triggerfile
468         echo "  significance:  "${results[$num+18]}" sigma" >> $triggerfile
469         echo "  cu:            "${results[$num+19]}" +- "${results[$num+20]}" CU" >> $triggerfile
470         echo "  corr. cu:      "${results[$num+21]}" +- "${results[$num+22]}" CU" >> $triggerfile
471         echo "  zd:            "${results[$num+23]}" - "${results[$num+24]}" degree" >> $triggerfile
472         echo "  th:            "${results[$num+25]}" - "${results[$num+26]}" DAC counts" >> $triggerfile
473         # additional information fast rise/decay trigger
474         if [ $triggertype -eq 3 ]
475         then
476            echo "Flux doubling/halfing times: " >> $triggerfile
477            echo "---------------------------- " >> $triggerfile
478            echo "  excess: "$exc >> $triggerfile
479            echo "  excess old: "$excold >> $triggerfile
480            echo "  significance: "$sig >> $triggerfile
481            echo "  significance old: "$sigprev >> $triggerfile
482            echo "  slope: "$excold >> $triggerfile
483            echo "  slope old: "$slopeold >> $triggerfile
484            echo "  excess 60 min: "${results2[19]}  >> $triggerfile
485            echo "  excess 60 min old: "${results2[27+19]} >> $triggerfile
486            echo "  slope 60 min: "$slope60 >> $triggerfile
487            echo "  significance 60 min: "$sig60 >> $triggerfile
488         fi
489         
490         if [ -e $donetriggerfile ]
491         then
492            diff $donetriggerfile $triggerfile >/dev/null
493            checkstatus=`echo $?`
494            if [ $checkstatus -gt 0 ]
495            then
496               # keep history of non-sent triggers
497               donetriggerfile2=$donetriggerfile"-changed-"`date +%Y%m%d%H%M%S`
498               cp $donetriggerfile $donetriggerfile2
499            fi
500         fi
501         # missing: get summary of whole observation
502         
503         # send email only of $donetriggerfile doesn't exists
504         if ! [ -e $donetriggerfile ] 
505         then 
506            query="INSERT FlareAlerts.FlareTriggers SET fTriggerInserted=Now(), fNight="$night", fRunID="$runid", fTriggerType="$triggertype
507            if [ "$bin" = "" ]
508            then
509               query=$query", fBinning=NULL"
510            else
511               query=$query", fBinning="$bin
512            fi
513            echo $query
514            sendquery >> $logfile
515           
516            # AMON case: create VOEvent-File
517            if [ $triggertype -eq 5 ]
518            then
519               voeventfile=$voeventpath"/"`basename $triggerfile`".xml"
520               # missing: check for archive file
521               #   if exist -> do revision
522               echo "creating "$voeventfile
523               echo "Creating "$voeventfile >> $logfile
524               #print_voevent_file
525               print_voevent_file > $voeventfile
526               # in amon-case no email needs to be sent
527            else
528               if [ "$bin" = "" ] && [ $trigger2 -eq 0 ]
529               then
530                  continue
531               fi
532               echo "sending["$triggertype"] "$triggerfile
533               echo "sending["$triggertype"] "$triggerfile >> $logfile
534               cat $triggerfile | mail -s 'test flare alert ' -b $emailfrom -r $emailfrom $emailto
535               #cat $triggerfile | mail -s "test flare alert for $sourcename " $emailto
536               # that's also the cases for making a call
537               # fill DB for shifthelper
538            fi
539         fi
540      fi
541
542      # counter
543      i=`echo $i +1 | bc -l`
544   done
545   
546   echo "  found "$i" data point(s)." >> $logfile
547   echo "" >> $logfile
548}
549
550get_average_flux()
551{
552   # query average flux from DB
553   query="SELECT ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) as excrate,  "
554   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excerr, "
555   query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime "
556   query=$query"FROM AnalysisResultsRunLP "
557   query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
558   if [ "$1" = "" ]
559   then 
560      query=$query"WHERE fSourceKey="$sourcekey" AND NOT ISNULL(fNumExcEvts) "
561   else
562      query=$query"WHERE fSourceKey="$sourcekey" AND fNight BETWEEN "$1" AND "$2" AND NOT ISNULL(fNumExcEvts) "
563   fi
564   query=$query"GROUP BY fSourceKey "
565   sendquery
566   # missing - get in CU to correct for fluctuations of CU
567}
568
569
570
571# main part of the script
572
573# missing: check if twistd client is running
574#   + check if there are remaining files in the to-send-folder of amon
575# -> send email if amon connection has problem
576
577
578for sourcekey in ${sourcekeys[@]}
579do
580   query="SELECT fSourceName FROM Source WHERE fSourceKey="$sourcekey
581   sourcename=`sendquery` #do not combine this with other source info as sourcename can have spaces
582   
583   # todo: what about data check ?
584   # should avg include current night?
585   total=( `get_average_flux` )
586   month=( `get_average_flux \`date -d $night' - 1 MONTH' +%Y%m%d\` $night` )
587   year=( `get_average_flux \`date -d $night' - 1 YEAR' +%Y%m%d\` $night` )
588   avgflux=${total[0]}
589   avgfluxmonth=${month[0]}
590   avgfluxyear=${year[0]}
591   error=${total[1]}
592   errormonth=${month[1]}
593   erroryear=${year[1]}
594   
595   # getting some information on the source
596   query="SELECT fDeclination, fRightAscension, fRightAscension/24.*15 FROM Source WHERE fSourceKey="$sourcekey
597   sourceinfo=( `sendquery` )
598
599   # ignore Crab
600   if [ $sourcekey -eq 5 ]
601   then
602      continue
603   fi
604   printprocesslog "INFO Evaluation for $sourcename ... "
605   echo "Evaluation for $sourcename ... " >> $logfile
606   echo " average fluxes: "$avgfluxmonth"+-"$errormonth" evts/h (last month) "$avgfluxyear"+-"$erroryear" evts/h (last year) "$avgflux"+-"$error" evts/h (all)" >> $logfile
607   
608   # missing: get limits from DB (structure needs to be defined)
609   
610   # triggers in the frame of the MoU in the gamma-ray community
611   triggertype=1
612   # limits
613   siglimit=3.0 # sigma
614   #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ]
615   if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 5 ]
616   then
617      exclimit=3.0 # CU
618   else
619      exclimit=0.5 # CU
620   fi
621   # only if rate goes even higher, we have to react
622   onlyifhigher="yes"
623   higher=1
624   
625   printprocesslog "INFO checking for [General gamma-ray MoU]" >> $logfile
626   echo "[General gamma-ray MoU]" >> $logfile
627   echo " nightly binning..." >> $logfile
628   # checking nightly binning
629   bin=
630   get_query_nightly_binning
631   results=( `sendquery` )
632   evaluate_result
633   
634   # 20 min binning
635   bin=20
636   echo " "$bin" min binning..." >> $logfile
637   get_query_minute_binning $bin
638   results=( `sendquery` )
639   evaluate_result
640   
641   # triggers to MAGIC
642   # Mrk 501 proposal
643   triggertype=2
644   if [ $sourcekey -eq 2 ]
645   then 
646      siglimit=3.0
647      exclimit=2.0 # cu
648      printprocesslog "INFO checking for [Trigger to MAGIC 501 proposal]" >> $logfile
649      echo "[Trigger to MAGIC 501 proposal]" >> $logfile
650      # checking nightly binning
651      bin=
652      echo " nightly binning..." >> $logfile
653      get_query_nightly_binning
654      results=( `sendquery` )
655      evaluate_result
656      # 20 min binning
657      bin=20
658      echo " "$bin" min binning..." >> $logfile
659      get_query_minute_binning $bin
660      results=( `sendquery` )
661      evaluate_result
662   fi
663   # Mother of ToO - fast rise/decay
664   # sources: Mrk 421, Mrk 501, 2344, 1959
665   triggertype=3
666   #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ] || [ $sourcekey -eq 5 ] # for testing
667   if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ]
668   then 
669      printprocesslog "INFO checking for [Trigger to MAGIC - fast rise/decay]" >> $logfile
670      echo "[Trigger to MAGIC - fast rise/decay]" >> $logfile
671      ## keep thresholds low (or do not use in evaluation)
672      #siglimit=2.0
673      #exclimit=0.5
674      # limits in slope
675      slopelimit=1.0 # 1CU/h
676      siglimit=3.0 # 1 sigma in 1 hour
677      # binning
678      bin=30
679      echo " "$bin" min binning..." >> $logfile
680      get_query_minute_binning $bin
681      results=( `sendquery` )
682      evaluate_result
683   fi
684   
685   # X-ray ToO
686   triggertype=4
687   # to be added
688   
689   # AMON - automatic triggers using VOEvent files
690   triggertype=5
691   printprocesslog "INFO checking for [Trigger to AMON]" >> $logfile
692   echo "[Trigger to AMON]" >> $logfile
693   echo " details still to be defined" >> $logfile
694   # missing: trigger limits and binning still to be defined
695   #   x times above average + significance limit ?
696   #   sub-threshold?
697   # FP-rate to be calculated
698   # use for the moment 0.5 CU and 3 sigma
699   siglimit=3.0
700   exclimit=0.5
701   onlyifhigher="no"
702   # checking nightly binning
703   bin=
704   echo " nightly binning..." >> $logfile
705   get_query_nightly_binning
706   results=( `sendquery` )
707   evaluate_result
708   bin=20
709   echo " "$bin" min binning..." >> $logfile
710   get_query_minute_binning $bin
711   results=( `sendquery` )
712   evaluate_result
713   
714   echo "" >> $logfile
715   echo "" >> $logfile
716done
717
718finish
719
720# for archival testing:
721for (( i=0; i < 100 ; i++))
722do 
723   date=`date --date="-${i}days" +%Y%m%d`
724   /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date
725done
726
727
Note: See TracBrowser for help on using the repository browser.