source: trunk/DataCheck/QuickLook/FlareAlerts.sh @ 19473

Last change on this file since 19473 was 19473, checked in by dorner, 4 months ago
fixed query of CU
  • Property svn:executable set to *
File size: 30.1 KB
Line 
1#!/bin/bash
2#
3
4# missing:
5# finalize DB-entries for being called
6# prepare directly template for email
7#  maybe prepare website with all necessary links and information
8# add x-ray trigger as trigger type or
9#    add note for values > 70 evts/h for X-ray triggers
10#
11# add more information:
12#   mjd, obs-summary, weather info (clouds? dust?), CU, corrected excrates
13#
14# evaluate output of all nights (check also for crab to estimate fluctuations)
15# error emails in case no DB content / no QLA
16# calculate delay of QLA and send email if > 30 Min
17
18# run for more nights:
19# for (( i=0; i < 70 ; i++)); do date=`date --date="-${i}days" +%Y%m%d`; /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date; done > logfile
20
21source `dirname $0`/../Sourcefile.sh
22printprocesslog "INFO starting $0"
23
24emailfrom=dorner@astro.uni-wuerzburg.de
25emailto=fact-online@lists.phys.ethz.ch
26
27if [ ! -e $flarealertspath ] || [ "$flarealertspath" == "" ]
28then 
29   echo "flarealertspath "$flarealertspath" missing on "$HOSTNAME
30   printprocesslog "ERROR flarealertspath "$flarealertspath" missing on "$HOSTNAME
31   finish
32fi
33voeventpath="/home/fact/amon/flare_alerts/"
34
35logfile=$runlogpath"/FlareAlerts-"$datetime".log"
36date > $logfile
37
38# get date
39if [ "$1" != "" ]
40then
41   checkstring=`echo $1 | grep -E -o '^20[0-9][0-9][01][0-9][0-3][0-9]$'`
42   echo $checkstring
43   if [ "$checkstring" = "" ]
44   then
45      night=`date +%Y%m%d --date="-12 HOUR"`
46   else
47      night=$1
48   fi
49else
50   night=`date +%Y%m%d --date="-12 HOUR"`
51fi
52
53echo "Processing "$night >> $logfile
54
55
56# get sources for current night from DB (RunInfo)
57query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunTypeKey=1 AND NOT ISNULL(fSourceKey) GROUP BY fSourceKey"
58sourcekeys=( `sendquery` )
59if [ ${#sourcekeys[@]} -eq 0 ]
60then
61   echo " No sources found for "$night >> $logfile
62   finish
63fi
64
65printprocesslog "INFO Checking the "${#sourcekeys[@]}" sourcekeys: "${sourcekeys[@]}
66
67# some stuff for queries:
68ontime="IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn)"
69threshold="IF (ISNULL(fThresholdMinSet), fThresholdMedian, fThresholdMinSet)"
70#cu="20.0"
71#cu="CUQLA(fNight)"
72cu="fCU"
73corr="1"
74# missing: corrected excessrates
75
76
77# missing: excerr is NOT rate
78function get_query_nightly_binning()
79{
80   # query to get information from DB
81   query="SELECT fSourceKey AS num, "
82   query=$query"fNight AS night, MIN(fRunID) AS runmin, MAX(fRunID) AS runmax, "
83   query=$query"MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
84   query=$query"ROUND(SUM("$ontime")/3600.,2) AS ontime, "
85   query=$query"SUM(fNumSigEvts) AS sig, SUM(fNumBgEvts) AS bg, "
86   query=$query"ROUND(SUM(fNumBgEvts)/SUM("$ontime")*3600,2) AS bgrate, "
87   query=$query"SUM(fNumExcEvts) AS exc, "
88   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts)), 2) AS excerr, "
89   query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,2) AS excrate, "
90   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 2) AS excrateerr, "
91   query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600,2) as corexcrate, " # put here correction factor
92   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr"), 2) AS corexcrateerr, " # correctionfactor = exc / exc_cor, put here correction factor
93   query=$query"ROUND(LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)),2) AS signif, "
94   query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600/"$cu",2) AS cu, " # make value time dependent
95   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600/"$cu", 2) AS cuerr, " # make value time dependent
96   query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600/"$cu",2) as corcu, " # make value time dependent # put here correction factor
97   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr")/"$cu", 2) AS corcuerr, " # correctionfactor = exc / exc_cor # make value time dependent # put here correction factor
98   query=$query"MIN(fZenithDistanceMin) as zdmin, MAX(fZenithDistanceMax) as zdmax, "
99   query=$query"MIN("$threshold") as thmin, MAX("$threshold") as thmax "
100   query=$query"FROM AnalysisResultsRunLP "
101   query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
102   query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
103   query=$query"GROUP BY fNight, fSourceKey "
104#   query=$query"ORDER BY fRunStart "
105   query=$query"HAVING ontime > 0.5 " # at least 30 minutes of observation
106}
107
108function get_query_minute_binning()
109{
110   # set binning
111   if [ "$1" != "" ]
112   then
113      bin2=$1
114   else
115      bin2=$bin
116   fi
117   # query to get information from DB
118   query="SELECT MAX(o.b) AS num, "
119   query=$query"MIN(o.n) AS night, MIN(o.run) AS runmin, MAX(o.run) AS runmax, "
120   query=$query"MIN(o.start) AS start, MAX(o.stop) AS stop, "
121   query=$query"ROUND(SUM(o.ot)/60.,2) AS ontime, "
122   query=$query"SUM(o.sig) AS sig, SUM(o.bg) AS bg, "
123   query=$query"ROUND(SUM(o.bg)/SUM(o.ot)*3600,2) AS bgrate, "
124   query=$query"SUM(o.exc) AS exc, "
125   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg)), 2) AS excerr, "
126   query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600,2) AS excrate, "
127   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600, 2) AS excrateerr, "
128   query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600,2) as corexcrate, "
129   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor), 2) AS corexcrateerr, " # correctionfactor = exc / exc_cor
130   query=$query"ROUND(LiMa(SUM(o.sig), SUM(o.bg)),2) AS signif, "
131   query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600/AVG(o.cu),2) AS cu, "
132   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600/Avg(o.cu), 2) AS cuerr, "
133   query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600/Avg(o.cu),2) as corcu, "
134   query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor)/Avg(o.cu), 2) AS corcuerr, " # correctionfactor = exc / exc_cor
135   query=$query"MIN(o.zdmin) as zdmin, MAX(o.zdmax) as zdmax, "
136   query=$query"MIN(o.th) as thmin, MAX(o.th) as thmax "
137   query=$query"FROM ("
138   query=$query"SELECT "
139   query=$query"fRunID AS run, fNight AS n, "
140   query=$query"@ot:= "$ontime" AS ot, "
141   query=$query"fRunStart AS start, fRunStop AS stop, "
142   query=$query"fNumExcEvts AS exc, fNumBgEvts AS bg, fNumSigEvts AS sig, "
143   query=$query"fNumExcEvts*"$corr" AS exccor, " # put here correction factor
144   query=$query$cu" as cu, " # make value time dependent
145   query=$query"fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "
146   query=$query$threshold" AS th, "
147   query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @bl, @bl := @bl + 1) AS b,  "
148   query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @"
149   query=$query"night :=fNight AS night FROM AnalysisResultsRunLP "
150   query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
151   query=$query"CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
152   query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
153#   if [ "$1" != "" ]
154#   then
155#      query=$query" AND fRunID <="${results[$num+3]}
156#   fi
157   query=$query" ORDER BY fRunStart "
158#   if [ "$1" != "" ]
159#   then
160#      query=$query"DESC"
161#   fi
162   query=$query" ) o GROUP BY b HAVING ontime > "$bin2"*0.75 ORDER BY start "
163#   if [ "$1" != "" ]
164#   then
165#      query=$query"DESC"
166#   fi
167   #echo $query
168}
169
170function print_voevent_file()
171{
172   # put here voevent file
173   # make sure that it is written to amon-folder
174   echo '<?xml version="1.0" ?>'
175   echo '<voe:VOEvent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
176   echo 'xmlns:voe="http://www.ivoa.net/xml/VOEvent/v2.0"'
177   echo 'xsi:schemaLocation="http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd"'
178   echo ' version="2.0" role="test" ivorn="ivo://amon/fact_#5_3772_0">'
179   echo '    <Who>'
180   echo '        <AuthorIVORN>ivo://FACT</AuthorIVORN>'
181   echo '        <Date>'`date +%F\ %H:%M:%S`'</Date>'
182   echo '    </Who>'
183   # FACT = 5 in AMON
184   echo '    <What>'
185   echo '        <Param name="stream" dataType="float" value="5" ucd="meta.number" unit=" ">'
186   echo '            <Description>Stream number</Description>'
187   echo '        </Param>'
188   # event identifier
189   # nightly binning: night+sourcekey
190   # 20 min binning: night+source+?
191   echo '        <Param name="id" dataType="float" value="'${night}`printf %03d $runid`${sourcekey}'" ucd="meta.number" unit=" ">'
192   echo '            <Description>Id number</Description>'
193   echo '        </Param>'
194   # keep 0 for the beginning
195   # how to handle updates? e.g. in case of full disk and random processing
196   echo '        <Param name="rev" dataType="float" value="0" ucd="meta.number" unit=" ">'
197   echo '            <Description>Revision number</Description>'
198   echo '        </Param>'
199   # what to do with this? in IC it's number of neutrinos, but does number of gammas make sense?
200   #  (excevts? depends on time range...)
201   echo '        <Param name="nevents" dataType="float" value="1" ucd="meta.number" unit=" ">'
202   echo '            <Description>Number of events</Description>'
203   echo '        </Param>'
204   # get time window (stop of last - start of first run)
205   # maybe: don't send alert if time window > xxx
206   echo '        <Param name="deltaT" dataType="float" value="0.0" ucd="time.duration" unit="s">'
207   echo '            <Description>Time window of the burst</Description>'
208   echo '        </Param>'
209   # probably 0
210   echo '        <Param name="sigmaT" dataType="float" value="0.0" ucd="time" unit="s">'
211   echo '            <Description>Uncertainty of the time window</Description>'
212   echo '        </Param>'
213   # FPR - to be calculated
214   echo '        <Param name="false_pos" dataType="float" value="-1" ucd="stat.probability" unit="s-1.sr-1 ">'
215   echo '            <Description>False positive rate</Description>'
216   echo '        </Param>'
217   # some significane value - how significant is the alert?
218   # to be discussed and calculated
219   echo '        <Param name="pvalue" dataType="float" value="1.0" ucd="stat.probability" unit=" ">'
220   echo '            <Description>Pvalue</Description>'
221   echo '        </Param>'
222   # use source RA/Dec? but then naming doesn't fit
223   # -> leave empty for the moment
224   echo '        <Param name="point_RA" dataType="float" value="-1.0" ucd="os.eq.ra" unit="deg">'
225   echo '            <Description>Pointing RA</Description>'
226   echo '        </Param>'
227   echo '        <Param name="point_dec" dataType="float" value="-1.0" ucd="os.eq.dec" unit="deg">'
228   echo '            <Description>Pointing Dec</Description>'
229   echo '        </Param>'
230   # shape of psf - to be determined - fit thetaplot? for crab or mc
231   echo '        <Param name="psf_type" dataType="string" value="fisher" ucd="meta.code.multip" unit=" ">'
232   echo '            <Description>Type of psf (skymap, fisher, kent, king)</Description>'
233   echo '        </Param>'
234   echo '        <Group name="aux_params">'
235   # which other information to add?
236   echo '            <Param name="zenith" dataType="float" value="0.0" ucd="os.lc.ze" unit="deg"/>'
237   echo '            <Param name="xyz" dataType="float" value="0.0" ucd="os.lc.ze" unit="xyz"/>'
238   echo '        </Group>'
239   echo '    </What>'
240   echo '    <WhereWhen>'
241   echo '        <ObsDataLocation>'
242   echo '            <ObservatoryLocation>'
243   echo '                <AstroCoordSystem id="UTC-GEOD-TOPO"/>'
244   echo '                <AstroCoords coord_system_id="UTC-GEOD-TOPO">'
245   echo '                    <Position3D unit="deg-deg-m">'
246   echo '                        <Name1>longitude</Name1>'
247   echo '                        <Name2>latitude</Name2>'
248   echo '                        <Name3>elevation</Name3>'
249   echo '                        <Value3>'
250   echo '                            <C1>-17.88</C1>'
251   echo '                            <C2>28.76</C2>'
252   echo '                            <C3>2200</C3>'
253   echo '                        </Value3>'
254   echo '                    </Position3D>'
255   echo '                </AstroCoords>'
256   echo '            </ObservatoryLocation>'
257   echo '            <ObservationLocation>'
258   echo '                <AstroCoordSystem id="UTC-ICRS-TOPO"/>'
259   echo '                <AstroCoords coord_system_id="UTC-ICRS-TOPO">'
260   echo '                    <Time unit="s">'
261   echo '                        <TimeInstant>'
262   # python needs the .0 in the time-format - as I don't have the time more accurately here, I put .0
263   echo '                            <ISOTime>'${results[$num+4]}' '${results[$num+5]}'.0</ISOTime>'
264   echo '                        </TimeInstant>'
265   echo '                    </Time>'
266   echo '                    <Position2D unit="deg-deg">'
267   echo '                        <Name1>RA</Name1>'
268   echo '                        <Name2>Dec</Name2>'
269   echo '                        <Value2>'
270   echo '                            <C1>'${sourceinfo[2]}'</C1>' # RA in deg
271   echo '                            <C2>'${sourceinfo[0]}'</C2>' # decl in deg
272   echo '                        </Value2>'
273   echo '                        <Error2Radius>0.1</Error2Radius>' # PSF of FACT
274   echo '                    </Position2D>'
275   echo '                </AstroCoords>'
276   echo '            </ObservationLocation>'
277   echo '        </ObsDataLocation>'
278   echo '    </WhereWhen>'
279   echo '    <Description>FACT flare event information</Description>'
280   echo '</voe:VOEvent>'
281   
282}
283
284function evaluate_result()
285{
286   oldexc=0
287   oldnightlyexc=0
288   exc=0
289   excold=0
290   slope=0
291   slopeprev=0
292   i=0
293   # be careful with start and stop (space inbetween) -> 27 columns instead of 25
294   while [ 0 -lt 1 ]
295   do
296      trigger=0
297      num=`echo "$i * 27" | bc -l`
298      if [ "${results[$num]}" = "" ]
299      then
300         break
301      fi
302      #night=${results[$num+1]}
303      runid=${results[$num+2]}
304      sig=${results[$num+18]} # significance
305      #exc=${results[$num+14]} # excrate
306      excold=$exc
307      exc=${results[$num+19]} # excrate in CU
308     
309      if [ "$onlyifhigher" = "yes" ]
310      then
311         higher=` echo " $exc > $oldexc " | bc `
312         if [ $higher -eq 1 ]
313         then
314            # keep old value
315            oldexc=$exc
316         fi
317      fi
318     
319      if [ "$bin" = "" ]   
320      then 
321         echo "  ontime: "${results[$num+8]}" h" >> $logfile
322      fi
323
324      # fast rise/decay trigger
325      #  this trigger type currently doesn't handle nightly binning
326      if [ $triggertype -eq 3 ]
327      then
328         slopeprev=$slope
329         sigprev=$sig
330         if [ "$excold" = "0" ]
331         then
332            slope=0
333         else
334            slope=`echo " scale=1; ( $exc - $excold ) / ( $bin / 60. ) " | bc -l `
335         fi
336         get_query_minute_binning 60
337         #echo $query
338         results2=( `sendquery` )
339         # need to check last hour backward
340         if [ "${results2[19]}" = "" ]
341         then 
342            slope60=0
343            sig60=0
344         else
345            sig60=${results2[18]} # significance 1h
346            if [ "${results2[27+19]}" = "" ]
347            then 
348               # maybe treat this case differently
349               slope60=${results2[19]}
350            else
351               slope60=`echo " ${results2[19]} - ${results2[27+19]} " | bc -l` # ie /1h
352            fi
353         fi
354         #echo "exc "$exc" excold "$excold
355         #echo "slope "$slope" prev "$slopeprev" sig "$sig" prevsig "$sigprev" sig60 "$sig60" slope60 "$slope60
356      fi
357      # missing: probably one should check also 20 min binning (s example 20170103)
358      # missing: check on still available observation time
359
360      echo "    "$i" "${results[$num+2]}"-"${results[$num+3]}"["${results[$num+8]}"] "$exc" "$sig >> $logfile
361     
362      case $triggertype in
363         1)   #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
364              trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
365              limits=$exclimit"_"$siglimit
366              ;;
367         2)   #echo "magic 501: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
368              trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
369              limits=$exclimit"_"$siglimit
370              ;;
371         3)   #echo "magic fast rise/decay"
372              #trigger=`echo " $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit && $sig >= $siglimit " | bc -l`
373              trigger=`echo " ( ( $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit ) || ( - $slope >= $slopelimit && - $slopeprev >= $slopelimit && - $slope60 >= $slopelimit  ) ) && $sig >= $siglimit " | bc -l`
374              limits=$slopelimit"_"$siglimit
375              ;;
376         5)   #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
377              trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
378              limits=$exclimit"_"$siglimit
379              ;;
380         *)   echo $triggertype" not yet implemented" >> $logfile
381              ;;
382      esac
383
384      # missing: maybe use $donetriggerfile for all cases - update of nightly case can be triggered differently
385      # missing: adapt triggerfilename for type 3 (slopelimit)
386      if [ $trigger -eq 1 ]
387      then
388         # do whatever to be done to trigger
389         #  - send email/sms / call
390         #  - create amon file
391         #  - prepare email for alert
392         #  - entry in DB
393         
394         if [ "$bin" = "" ]
395         then 
396            # nightly file: simply overwrite, but do not send trigger again
397            # or check value if it's increasing?
398            # missing: get information from previous trigger + interpret
399            triggerfile=$flarealertspath"/"$night"-"$limits"-source"$sourcekey".trigger"$triggertype
400         else
401            # smaller binning: check if trigger is identical
402            triggerfile=$flarealertspath"/"$night"_"`printf %03d $runid`"-"$limits"-source"$sourcekey".trigger"$triggertype
403         fi
404         donetriggerfile=$triggerfile".done"
405         #ls $donetriggerfile
406
407         # check if nightly flux increased
408         if [ "$bin" = "" ]
409         then 
410            trigger2=1
411            changedfiles=( `ls $donetriggerfile"-changed-"* 2>/dev/null` )
412            if [ ${#changedfiles[@]} -gt 0 ]
413            then 
414               oldnightlyexc=`grep "corr. cu:      [0-9].[0-9] " ${changedfiles[@]} | grep -o -E ' [0-9].[0-9] ' | sort | tail -1`
415               if [ "$oldnightlyexc" = "" ]
416               then
417                  oldnightlyexc=-1000
418               fi
419               trigger2=`echo " $exc > $oldnightlyexc " | bc -l`
420               #echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")"
421               echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")" >> $logfile
422            fi
423         fi
424         
425         # write new file only if old files do not agree
426         if [ -e $donetriggerfile ]
427         then
428            diff $donetriggerfile $triggerfile >/dev/null
429            checkstatus=`echo $?`
430            if [ $checkstatus -eq 0 ]
431            then
432               echo "  alert already done "$donetriggerfile >> $logfile
433               i=`echo $i +1 | bc -l`
434               continue
435            fi
436         fi
437         if [ -e $triggerfile ]
438         then
439            mv $triggerfile $donetriggerfile
440         fi
441         
442         #echo $night"_"$runid" "$sourcekey" -> "$triggerfile
443         echo "  writing "$triggerfile >> $logfile
444         touch $triggerfile
445         echo "Trigger found: " > $triggerfile
446         echo "-------------- " >> $triggerfile
447         echo "  type: "$triggertype >> $triggerfile
448         echo "  excess limit: "$exclimit" CU" >> $triggerfile
449         echo "  significance limit: "$siglimit" sigma" >> $triggerfile
450         if [ "$bin" = "" ]   
451         then 
452            echo "  nightly binning " >> $triggerfile
453         else
454            echo "  binning: "$bin" min" >> $triggerfile
455         fi
456         echo "Summary of flare event: " >> $triggerfile
457         echo "----------------------- " >> $triggerfile
458         echo "  source:        "$sourcename >> $triggerfile
459         echo "  night:         "${results[$num+1]} >> $triggerfile
460         echo "  runs:          "${results[$num+2]}" - "${results[$num+3]} >> $triggerfile
461         echo "  start:         "${results[$num+4]}" "${results[$num+5]}" UTC" >> $triggerfile
462         echo "  stop:          "${results[$num+6]}" "${results[$num+7]}" UTC" >> $triggerfile
463         if [ "$bin" = "" ]
464         then 
465            echo "  ontime:        "${results[$num+8]}" h" >> $triggerfile
466         else
467            echo "  ontime:        "${results[$num+8]}" min" >> $triggerfile
468         fi
469         #echo "  ontime:        "`echo "scale=1; ${results[$num+8]} / 60. " | bc -l`" min" #scale doesn't round properly
470         echo "  signal:        "${results[$num+9]}" evts" >> $triggerfile
471         echo "  background:    "${results[$num+10]}" evts" >> $triggerfile
472         echo "  bgrate:        "${results[$num+11]}" evts/h" >> $triggerfile
473         echo "  exc:           "${results[$num+12]}" +- "${results[$num+13]}" evts" >> $triggerfile
474         echo "  excrate:       "${results[$num+14]}" +- "${results[$num+15]}" evts/h" >> $triggerfile
475         echo "  corr. excrate: "${results[$num+16]}" - "${results[$num+17]}" evts/h" >> $triggerfile
476         echo "  significance:  "${results[$num+18]}" sigma" >> $triggerfile
477         echo "  cu:            "${results[$num+19]}" +- "${results[$num+20]}" CU" >> $triggerfile
478         echo "  corr. cu:      "${results[$num+21]}" +- "${results[$num+22]}" CU" >> $triggerfile
479         echo "  zd:            "${results[$num+23]}" - "${results[$num+24]}" degree" >> $triggerfile
480         echo "  th:            "${results[$num+25]}" - "${results[$num+26]}" DAC counts" >> $triggerfile
481         # additional information fast rise/decay trigger
482         if [ $triggertype -eq 3 ]
483         then
484            echo "Flux doubling/halfing times: " >> $triggerfile
485            echo "---------------------------- " >> $triggerfile
486            echo "  excess: "$exc >> $triggerfile
487            echo "  excess old: "$excold >> $triggerfile
488            echo "  significance: "$sig >> $triggerfile
489            echo "  significance old: "$sigprev >> $triggerfile
490            echo "  slope: "$excold >> $triggerfile
491            echo "  slope old: "$slopeold >> $triggerfile
492            echo "  excess 60 min: "${results2[19]}  >> $triggerfile
493            echo "  excess 60 min old: "${results2[27+19]} >> $triggerfile
494            echo "  slope 60 min: "$slope60 >> $triggerfile
495            echo "  significance 60 min: "$sig60 >> $triggerfile
496         fi
497         
498         if [ -e $donetriggerfile ]
499         then
500            diff $donetriggerfile $triggerfile >/dev/null
501            checkstatus=`echo $?`
502            if [ $checkstatus -gt 0 ]
503            then
504               # keep history of non-sent triggers
505               donetriggerfile2=$donetriggerfile"-changed-"`date +%Y%m%d%H%M%S`
506               cp $donetriggerfile $donetriggerfile2
507            fi
508         fi
509         # missing: get summary of whole observation
510         
511         # send email only of $donetriggerfile doesn't exists
512         if ! [ -e $donetriggerfile ] 
513         then 
514            query="INSERT FlareAlerts.FlareTriggers SET fTriggerInserted=Now(), fNight="$night", fRunID="$runid", fTriggerType="$triggertype", fSourceKey="$sourcekey
515            if [ "$bin" = "" ]
516            then
517               query=$query", fBinning=NULL"
518            else
519               query=$query", fBinning="$bin
520            fi
521            echo $query
522            sendquery >> $logfile
523           
524            # AMON case: create VOEvent-File
525            if [ $triggertype -eq 5 ]
526            then
527               voeventfile=$voeventpath"/"`basename $triggerfile`".xml"
528               # missing: check for archive file
529               #   if exist -> do revision
530               echo "creating "$voeventfile
531               echo "Creating "$voeventfile >> $logfile
532               #print_voevent_file
533               print_voevent_file > $voeventfile
534               # in amon-case no email needs to be sent
535            else
536               if [ "$bin" = "" ] && [ $trigger2 -eq 0 ]
537               then
538                  continue
539               fi
540               echo "sending["$triggertype"] "$triggerfile
541               echo "sending["$triggertype"] "$triggerfile >> $logfile
542               cat $triggerfile | mail -s 'flare alert' -b $emailfrom -r $emailfrom $emailto
543               #cat $triggerfile | mail -s "test flare alert for $sourcename " $emailto
544               # that's also the cases for making a call
545               # fill DB for shifthelper
546            fi
547         fi
548      fi
549
550      # counter
551      i=`echo $i +1 | bc -l`
552   done
553   
554   echo "  found "$i" data point(s)." >> $logfile
555   echo "" >> $logfile
556}
557
558get_average_flux()
559{
560   # query average flux from DB
561   query="SELECT ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,2) as excrate,  "
562   query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 2) AS excerr, "
563   query=$query"ROUND(SUM("$ontime")/3600.,2) AS ontime "
564   query=$query"FROM AnalysisResultsRunLP "
565   query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
566   if [ "$1" = "" ]
567   then 
568      query=$query"WHERE fSourceKey="$sourcekey" AND NOT ISNULL(fNumExcEvts) "
569   else
570      query=$query"WHERE fSourceKey="$sourcekey" AND fNight BETWEEN "$1" AND "$2" AND NOT ISNULL(fNumExcEvts) "
571   fi
572   query=$query"GROUP BY fSourceKey "
573   sendquery
574   # missing - get in CU to correct for fluctuations of CU
575}
576
577
578
579# main part of the script
580
581# missing: check if twistd client is running
582#   + check if there are remaining files in the to-send-folder of amon
583# -> send email if amon connection has problem
584
585
586for sourcekey in ${sourcekeys[@]}
587do
588   query="SELECT fSourceName FROM Source WHERE fSourceKey="$sourcekey
589   sourcename=`sendquery` #do not combine this with other source info as sourcename can have spaces
590   
591   # todo: what about data check ?
592   # should avg include current night?
593   total=( `get_average_flux` )
594   month=( `get_average_flux \`date -d $night' - 1 MONTH' +%Y%m%d\` $night` )
595   year=( `get_average_flux \`date -d $night' - 1 YEAR' +%Y%m%d\` $night` )
596   avgflux=${total[0]}
597   avgfluxmonth=${month[0]}
598   avgfluxyear=${year[0]}
599   error=${total[1]}
600   errormonth=${month[1]}
601   erroryear=${year[1]}
602   
603   # getting some information on the source
604   query="SELECT fDeclination, fRightAscension, fRightAscension/24.*15 FROM Source WHERE fSourceKey="$sourcekey
605   sourceinfo=( `sendquery` )
606
607   # ignore Crab
608   if [ $sourcekey -eq 5 ]
609   then
610      continue
611   fi
612   printprocesslog "INFO Evaluation for $sourcename ... "
613   echo "Evaluation for $sourcename ... " >> $logfile
614   echo " average fluxes: "$avgfluxmonth"+-"$errormonth" evts/h (last month) "$avgfluxyear"+-"$erroryear" evts/h (last year) "$avgflux"+-"$error" evts/h (all)" >> $logfile
615   
616   # missing: get limits from DB (structure needs to be defined)
617   
618   # triggers in the frame of the MoU in the gamma-ray community
619   triggertype=1
620   # limits
621   siglimit=3.0 # sigma
622   #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ]
623   if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 5 ]
624   then
625      exclimit=3.0 # CU
626   else
627      exclimit=0.5 # CU
628   fi
629   # only if rate goes even higher, we have to react
630   onlyifhigher="yes"
631   higher=1
632   
633   printprocesslog "INFO checking for [General gamma-ray MoU]" >> $logfile
634   echo "[General gamma-ray MoU]" >> $logfile
635   echo " nightly binning..." >> $logfile
636   # checking nightly binning
637   bin=
638   get_query_nightly_binning
639   results=( `sendquery` )
640   evaluate_result
641   
642   # 20 min binning
643   bin=20
644   echo " "$bin" min binning..." >> $logfile
645   get_query_minute_binning $bin
646   results=( `sendquery` )
647   evaluate_result
648   
649   # triggers to MAGIC
650   # Mrk 501 proposal
651   triggertype=2
652   if [ $sourcekey -eq 2 ]
653   then 
654      siglimit=3.0
655      exclimit=1.5 # cu
656      printprocesslog "INFO checking for [Trigger to MAGIC 501 proposal]" >> $logfile
657      echo "[Trigger to MAGIC 501 proposal]" >> $logfile
658      # checking nightly binning
659      bin=
660      echo " nightly binning..." >> $logfile
661      get_query_nightly_binning
662      results=( `sendquery` )
663      evaluate_result
664      # 20 min binning
665      bin=30
666      echo " "$bin" min binning..." >> $logfile
667      get_query_minute_binning $bin
668      results=( `sendquery` )
669      evaluate_result
670   fi
671   # Mother of ToO - fast rise/decay
672   # sources: Mrk 421, Mrk 501, 2344, 1959
673   triggertype=3
674   #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ] || [ $sourcekey -eq 5 ] # for testing
675   if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ]
676   then 
677      printprocesslog "INFO checking for [Trigger to MAGIC - fast rise/decay]" >> $logfile
678      echo "[Trigger to MAGIC - fast rise/decay]" >> $logfile
679      ## keep thresholds low (or do not use in evaluation)
680      #siglimit=2.0
681      #exclimit=0.5
682      # limits in slope
683      slopelimit=1.0 # 1CU/h
684      siglimit=3.0 # 1 sigma in 1 hour
685      # binning
686      bin=30
687      echo " "$bin" min binning..." >> $logfile
688      get_query_minute_binning $bin
689      results=( `sendquery` )
690      evaluate_result
691   fi
692   
693   # X-ray ToO
694   triggertype=4
695   # to be added
696   
697   # AMON - automatic triggers using VOEvent files
698   triggertype=5
699   printprocesslog "INFO checking for [Trigger to AMON]" >> $logfile
700   echo "[Trigger to AMON]" >> $logfile
701   echo " details still to be defined" >> $logfile
702   # missing: trigger limits and binning still to be defined
703   #   x times above average + significance limit ?
704   #   sub-threshold?
705   # FP-rate to be calculated
706   # use for the moment 0.5 CU and 3 sigma
707   siglimit=3.0
708   exclimit=0.5
709   onlyifhigher="no"
710   # checking nightly binning
711   bin=
712   echo " nightly binning..." >> $logfile
713   get_query_nightly_binning
714   results=( `sendquery` )
715   evaluate_result
716   bin=20
717   echo " "$bin" min binning..." >> $logfile
718   get_query_minute_binning $bin
719   results=( `sendquery` )
720   evaluate_result
721   
722   echo "" >> $logfile
723   echo "" >> $logfile
724done
725
726finish
727
728# for archival testing:
729for (( i=0; i < 100 ; i++))
730do 
731   date=`date --date="-${i}days" +%Y%m%d`
732   /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date
733done
734
735
Note: See TracBrowser for help on using the repository browser.