1 | #!/bin/bash
|
---|
2 | #
|
---|
3 |
|
---|
4 | # missing:
|
---|
5 | # finalize DB-entries for being called
|
---|
6 | # prepare directly template for email
|
---|
7 | # add x-ray trigger as trigger type or
|
---|
8 | # add note for values > 70 evts/h for X-ray triggers
|
---|
9 | #
|
---|
10 | # add more information:
|
---|
11 | # mjd, obs-summary, weather info (clouds? dust?), CU, corrected excrates
|
---|
12 | #
|
---|
13 | # evaluate output of all nights (check also for crab to estimate fluctuations)
|
---|
14 | # error emails in case no DB content / no QLA
|
---|
15 | # calculate delay of QLA and send email if > 30 Min
|
---|
16 |
|
---|
17 | # run for more nights:
|
---|
18 | # for (( i=0; i < 70 ; i++)); do date=`date --date="-${i}days" +%Y%m%d`; /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date; done > logfile
|
---|
19 |
|
---|
20 | source `dirname $0`/../Sourcefile.sh
|
---|
21 | printprocesslog "INFO starting $0"
|
---|
22 |
|
---|
23 | emailfrom=dorner@astro.uni-wuerzburg.de
|
---|
24 | emailto=fact-online@lists.phys.ethz.ch
|
---|
25 |
|
---|
26 | if [ ! -e $flarealertspath ] || [ "$flarealertspath" == "" ]
|
---|
27 | then
|
---|
28 | echo "flarealertspath "$flarealertspath" missing on "$HOSTNAME
|
---|
29 | printprocesslog "ERROR flarealertspath "$flarealertspath" missing on "$HOSTNAME
|
---|
30 | finish
|
---|
31 | fi
|
---|
32 | voeventpath="/home/fact/amon/flare_alerts/"
|
---|
33 |
|
---|
34 | logfile=$runlogpath"/FlareAlerts-"$datetime".log"
|
---|
35 | date > $logfile
|
---|
36 |
|
---|
37 | # get date
|
---|
38 | if [ "$1" != "" ]
|
---|
39 | then
|
---|
40 | checkstring=`echo $1 | grep -E -o '^20[0-9][0-9][01][0-9][0-3][0-9]$'`
|
---|
41 | echo $checkstring
|
---|
42 | if [ "$checkstring" = "" ]
|
---|
43 | then
|
---|
44 | night=`date +%Y%m%d --date="-12 HOUR"`
|
---|
45 | else
|
---|
46 | night=$1
|
---|
47 | fi
|
---|
48 | else
|
---|
49 | night=`date +%Y%m%d --date="-12 HOUR"`
|
---|
50 | fi
|
---|
51 |
|
---|
52 | echo "Processing "$night >> $logfile
|
---|
53 |
|
---|
54 |
|
---|
55 | # get sources for current night from DB (RunInfo)
|
---|
56 | query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunTypeKey=1 AND NOT ISNULL(fSourceKey) GROUP BY fSourceKey"
|
---|
57 | sourcekeys=( `sendquery` )
|
---|
58 | if [ ${#sourcekeys[@]} -eq 0 ]
|
---|
59 | then
|
---|
60 | echo " No sources found for "$night >> $logfile
|
---|
61 | finish
|
---|
62 | fi
|
---|
63 |
|
---|
64 | printprocesslog "INFO Checking the "${#sourcekeys[@]}" sourcekeys: "${sourcekeys[@]}
|
---|
65 |
|
---|
66 | # some stuff for queries:
|
---|
67 | ontime="IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn)"
|
---|
68 | threshold="IF (ISNULL(fThresholdMinSet), fThresholdMedian, fThresholdMinSet)"
|
---|
69 | #cu="20.0"
|
---|
70 | cu="CUQLA(fNight)"
|
---|
71 | corr="1"
|
---|
72 | # missing: corrected excessrates
|
---|
73 |
|
---|
74 |
|
---|
75 | # missing: excerr is NOT rate
|
---|
76 | function get_query_nightly_binning()
|
---|
77 | {
|
---|
78 | # query to get information from DB
|
---|
79 | query="SELECT fSourceKey AS num, "
|
---|
80 | query=$query"fNight AS night, MIN(fRunID) AS runmin, MAX(fRunID) AS runmax, "
|
---|
81 | query=$query"MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
|
---|
82 | query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime, "
|
---|
83 | query=$query"SUM(fNumSigEvts) AS sig, SUM(fNumBgEvts) AS bg, "
|
---|
84 | query=$query"ROUND(SUM(fNumBgEvts)/SUM("$ontime")*3600,1) AS bgrate, "
|
---|
85 | query=$query"SUM(fNumExcEvts) AS exc, "
|
---|
86 | query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts)), 1) AS excerr, "
|
---|
87 | query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) AS excrate, "
|
---|
88 | query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excrateerr, "
|
---|
89 | query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600,1) as corexcrate, " # put here correction factor
|
---|
90 | query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr"), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor, put here correction factor
|
---|
91 | query=$query"ROUND(LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)),1) AS signif, "
|
---|
92 | query=$query"ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600/"$cu",1) AS cu, " # make value time dependent
|
---|
93 | query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600/"$cu", 1) AS cuerr, " # make value time dependent
|
---|
94 | query=$query"ROUND(SUM(fNumExcEvts*"$corr")/SUM("$ontime")*3600/"$cu",1) as corcu, " # make value time dependent # put here correction factor
|
---|
95 | query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600*SUM(fNumExcEvts)/SUM(fNumExcEvts*"$corr")/"$cu", 1) AS corcuerr, " # correctionfactor = exc / exc_cor # make value time dependent # put here correction factor
|
---|
96 | query=$query"MIN(fZenithDistanceMin) as zdmin, MAX(fZenithDistanceMax) as zdmax, "
|
---|
97 | query=$query"MIN("$threshold") as thmin, MAX("$threshold") as thmax "
|
---|
98 | query=$query"FROM AnalysisResultsRunLP "
|
---|
99 | query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
|
---|
100 | query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
|
---|
101 | query=$query"GROUP BY fNight, fSourceKey "
|
---|
102 | # query=$query"ORDER BY fRunStart "
|
---|
103 | query=$query"HAVING ontime > 0.5 " # at least 30 minutes of observation
|
---|
104 | }
|
---|
105 |
|
---|
106 | function get_query_minute_binning()
|
---|
107 | {
|
---|
108 | # set binning
|
---|
109 | if [ "$1" != "" ]
|
---|
110 | then
|
---|
111 | bin2=$1
|
---|
112 | else
|
---|
113 | bin2=$bin
|
---|
114 | fi
|
---|
115 | # query to get information from DB
|
---|
116 | query="SELECT MAX(o.b) AS num, "
|
---|
117 | query=$query"MIN(o.n) AS night, MIN(o.run) AS runmin, MAX(o.run) AS runmax, "
|
---|
118 | query=$query"MIN(o.start) AS start, MAX(o.stop) AS stop, "
|
---|
119 | query=$query"ROUND(SUM(o.ot)/60.,1) AS ontime, "
|
---|
120 | query=$query"SUM(o.sig) AS sig, SUM(o.bg) AS bg, "
|
---|
121 | query=$query"ROUND(SUM(o.bg)/SUM(o.ot)*3600,1) AS bgrate, "
|
---|
122 | query=$query"SUM(o.exc) AS exc, "
|
---|
123 | query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg)), 1) AS excerr, "
|
---|
124 | query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600,1) AS excrate, "
|
---|
125 | query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600, 1) AS excrateerr, "
|
---|
126 | query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600,1) as corexcrate, "
|
---|
127 | query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor), 1) AS corexcrateerr, " # correctionfactor = exc / exc_cor
|
---|
128 | query=$query"ROUND(LiMa(SUM(o.sig), SUM(o.bg)),1) AS signif, "
|
---|
129 | query=$query"ROUND(SUM(o.exc)/SUM(o.ot)*3600/o.cu,1) AS cu, "
|
---|
130 | query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600/o.cu, 1) AS cuerr, "
|
---|
131 | query=$query"ROUND(SUM(o.exccor)/SUM(o.ot)*3600/o.cu,1) as corcu, "
|
---|
132 | query=$query"ROUND(ExcErr(SUM(o.sig), SUM(o.bg))/SUM(o.ot)*3600*SUM(o.exc)/SUM(o.exccor)/o.cu, 1) AS corcuerr, " # correctionfactor = exc / exc_cor
|
---|
133 | query=$query"MIN(o.zdmin) as zdmin, MAX(o.zdmax) as zdmax, "
|
---|
134 | query=$query"MIN(o.th) as thmin, MAX(o.th) as thmax "
|
---|
135 | query=$query"FROM ("
|
---|
136 | query=$query"SELECT "
|
---|
137 | query=$query"fRunID AS run, fNight AS n, "
|
---|
138 | query=$query"@ot:= "$ontime" AS ot, "
|
---|
139 | query=$query"fRunStart AS start, fRunStop AS stop, "
|
---|
140 | query=$query"fNumExcEvts AS exc, fNumBgEvts AS bg, fNumSigEvts AS sig, "
|
---|
141 | query=$query"fNumExcEvts*"$corr" AS exccor, " # put here correction factor
|
---|
142 | query=$query$cu" as cu, " # make value time dependent
|
---|
143 | query=$query"fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "
|
---|
144 | query=$query$threshold" AS th, "
|
---|
145 | query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @bl, @bl := @bl + 1) AS b, "
|
---|
146 | query=$query"IF (@night=fNight AND FLOOR((@os+@ot)/"$bin2"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @"
|
---|
147 | query=$query"night :=fNight AS night FROM AnalysisResultsRunLP "
|
---|
148 | query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
|
---|
149 | query=$query"CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
|
---|
150 | query=$query"WHERE fSourceKey="$sourcekey" AND fNight="$night" AND NOT ISNULL(fNumExcEvts) "
|
---|
151 | # if [ "$1" != "" ]
|
---|
152 | # then
|
---|
153 | # query=$query" AND fRunID <="${results[$num+3]}
|
---|
154 | # fi
|
---|
155 | query=$query" ORDER BY fRunStart "
|
---|
156 | # if [ "$1" != "" ]
|
---|
157 | # then
|
---|
158 | # query=$query"DESC"
|
---|
159 | # fi
|
---|
160 | query=$query" ) o GROUP BY b HAVING ontime > "$bin2"*0.75 ORDER BY start "
|
---|
161 | # if [ "$1" != "" ]
|
---|
162 | # then
|
---|
163 | # query=$query"DESC"
|
---|
164 | # fi
|
---|
165 | #echo $query
|
---|
166 | }
|
---|
167 |
|
---|
168 | function print_voevent_file()
|
---|
169 | {
|
---|
170 | # put here voevent file
|
---|
171 | # make sure that it is written to amon-folder
|
---|
172 | echo '<?xml version="1.0" ?>'
|
---|
173 | echo '<voe:VOEvent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
|
---|
174 | echo 'xmlns:voe="http://www.ivoa.net/xml/VOEvent/v2.0"'
|
---|
175 | echo 'xsi:schemaLocation="http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd"'
|
---|
176 | echo ' version="2.0" role="test" ivorn="ivo://amon/fact_#5_3772_0">'
|
---|
177 | echo ' <Who>'
|
---|
178 | echo ' <AuthorIVORN>ivo://FACT</AuthorIVORN>'
|
---|
179 | echo ' <Date>'`date +%F\ %H:%M:%S`'</Date>'
|
---|
180 | echo ' </Who>'
|
---|
181 | # FACT = 5 in AMON
|
---|
182 | echo ' <What>'
|
---|
183 | echo ' <Param name="stream" dataType="float" value="5" ucd="meta.number" unit=" ">'
|
---|
184 | echo ' <Description>Stream number</Description>'
|
---|
185 | echo ' </Param>'
|
---|
186 | # event identifier
|
---|
187 | # nightly binning: night+sourcekey
|
---|
188 | # 20 min binning: night+source+?
|
---|
189 | echo ' <Param name="id" dataType="float" value="'${night}`printf %03d $runid`${sourcekey}'" ucd="meta.number" unit=" ">'
|
---|
190 | echo ' <Description>Id number</Description>'
|
---|
191 | echo ' </Param>'
|
---|
192 | # keep 0 for the beginning
|
---|
193 | # how to handle updates? e.g. in case of full disk and random processing
|
---|
194 | echo ' <Param name="rev" dataType="float" value="0" ucd="meta.number" unit=" ">'
|
---|
195 | echo ' <Description>Revision number</Description>'
|
---|
196 | echo ' </Param>'
|
---|
197 | # what to do with this? in IC it's number of neutrinos, but does number of gammas make sense?
|
---|
198 | # (excevts? depends on time range...)
|
---|
199 | echo ' <Param name="nevents" dataType="float" value="1" ucd="meta.number" unit=" ">'
|
---|
200 | echo ' <Description>Number of events</Description>'
|
---|
201 | echo ' </Param>'
|
---|
202 | # get time window (stop of last - start of first run)
|
---|
203 | # maybe: don't send alert if time window > xxx
|
---|
204 | echo ' <Param name="deltaT" dataType="float" value="0.0" ucd="time.duration" unit="s">'
|
---|
205 | echo ' <Description>Time window of the burst</Description>'
|
---|
206 | echo ' </Param>'
|
---|
207 | # probably 0
|
---|
208 | echo ' <Param name="sigmaT" dataType="float" value="0.0" ucd="time" unit="s">'
|
---|
209 | echo ' <Description>Uncertainty of the time window</Description>'
|
---|
210 | echo ' </Param>'
|
---|
211 | # FPR - to be calculated
|
---|
212 | echo ' <Param name="false_pos" dataType="float" value="-1" ucd="stat.probability" unit="s-1.sr-1 ">'
|
---|
213 | echo ' <Description>False positive rate</Description>'
|
---|
214 | echo ' </Param>'
|
---|
215 | # some significane value - how significant is the alert?
|
---|
216 | # to be discussed and calculated
|
---|
217 | echo ' <Param name="pvalue" dataType="float" value="1.0" ucd="stat.probability" unit=" ">'
|
---|
218 | echo ' <Description>Pvalue</Description>'
|
---|
219 | echo ' </Param>'
|
---|
220 | # use source RA/Dec? but then naming doesn't fit
|
---|
221 | # -> leave empty for the moment
|
---|
222 | echo ' <Param name="point_RA" dataType="float" value="-1.0" ucd="os.eq.ra" unit="deg">'
|
---|
223 | echo ' <Description>Pointing RA</Description>'
|
---|
224 | echo ' </Param>'
|
---|
225 | echo ' <Param name="point_dec" dataType="float" value="-1.0" ucd="os.eq.dec" unit="deg">'
|
---|
226 | echo ' <Description>Pointing Dec</Description>'
|
---|
227 | echo ' </Param>'
|
---|
228 | # shape of psf - to be determined - fit thetaplot? for crab or mc
|
---|
229 | echo ' <Param name="psf_type" dataType="string" value="fisher" ucd="meta.code.multip" unit=" ">'
|
---|
230 | echo ' <Description>Type of psf (skymap, fisher, kent, king)</Description>'
|
---|
231 | echo ' </Param>'
|
---|
232 | echo ' <Group name="aux_params">'
|
---|
233 | # which other information to add?
|
---|
234 | echo ' <Param name="zenith" dataType="float" value="0.0" ucd="os.lc.ze" unit="deg"/>'
|
---|
235 | echo ' <Param name="xyz" dataType="float" value="0.0" ucd="os.lc.ze" unit="xyz"/>'
|
---|
236 | echo ' </Group>'
|
---|
237 | echo ' </What>'
|
---|
238 | echo ' <WhereWhen>'
|
---|
239 | echo ' <ObsDataLocation>'
|
---|
240 | echo ' <ObservatoryLocation>'
|
---|
241 | echo ' <AstroCoordSystem id="UTC-GEOD-TOPO"/>'
|
---|
242 | echo ' <AstroCoords coord_system_id="UTC-GEOD-TOPO">'
|
---|
243 | echo ' <Position3D unit="deg-deg-m">'
|
---|
244 | echo ' <Name1>longitude</Name1>'
|
---|
245 | echo ' <Name2>latitude</Name2>'
|
---|
246 | echo ' <Name3>elevation</Name3>'
|
---|
247 | echo ' <Value3>'
|
---|
248 | echo ' <C1>-17.88</C1>'
|
---|
249 | echo ' <C2>28.76</C2>'
|
---|
250 | echo ' <C3>2200</C3>'
|
---|
251 | echo ' </Value3>'
|
---|
252 | echo ' </Position3D>'
|
---|
253 | echo ' </AstroCoords>'
|
---|
254 | echo ' </ObservatoryLocation>'
|
---|
255 | echo ' <ObservationLocation>'
|
---|
256 | echo ' <AstroCoordSystem id="UTC-ICRS-TOPO"/>'
|
---|
257 | echo ' <AstroCoords coord_system_id="UTC-ICRS-TOPO">'
|
---|
258 | echo ' <Time unit="s">'
|
---|
259 | echo ' <TimeInstant>'
|
---|
260 | # python needs the .0 in the time-format - as I don't have the time more accurately here, I put .0
|
---|
261 | echo ' <ISOTime>'${results[$num+4]}' '${results[$num+5]}'.0</ISOTime>'
|
---|
262 | echo ' </TimeInstant>'
|
---|
263 | echo ' </Time>'
|
---|
264 | echo ' <Position2D unit="deg-deg">'
|
---|
265 | echo ' <Name1>RA</Name1>'
|
---|
266 | echo ' <Name2>Dec</Name2>'
|
---|
267 | echo ' <Value2>'
|
---|
268 | echo ' <C1>'${sourceinfo[2]}'</C1>' # RA in deg
|
---|
269 | echo ' <C2>'${sourceinfo[0]}'</C2>' # decl in deg
|
---|
270 | echo ' </Value2>'
|
---|
271 | echo ' <Error2Radius>0.1</Error2Radius>' # PSF of FACT
|
---|
272 | echo ' </Position2D>'
|
---|
273 | echo ' </AstroCoords>'
|
---|
274 | echo ' </ObservationLocation>'
|
---|
275 | echo ' </ObsDataLocation>'
|
---|
276 | echo ' </WhereWhen>'
|
---|
277 | echo ' <Description>FACT flare event information</Description>'
|
---|
278 | echo '</voe:VOEvent>'
|
---|
279 |
|
---|
280 | }
|
---|
281 |
|
---|
282 | function evaluate_result()
|
---|
283 | {
|
---|
284 | oldexc=0
|
---|
285 | exc=0
|
---|
286 | excold=0
|
---|
287 | slope=0
|
---|
288 | slopeprev=0
|
---|
289 | i=0
|
---|
290 | # be careful with start and stop (space inbetween) -> 27 columns instead of 25
|
---|
291 | while [ 0 -lt 1 ]
|
---|
292 | do
|
---|
293 | trigger=0
|
---|
294 | num=`echo "$i * 27" | bc -l`
|
---|
295 | if [ "${results[$num]}" = "" ]
|
---|
296 | then
|
---|
297 | break
|
---|
298 | fi
|
---|
299 | #night=${results[$num+1]}
|
---|
300 | runid=${results[$num+2]}
|
---|
301 | sig=${results[$num+18]} # significance
|
---|
302 | #exc=${results[$num+14]} # excrate
|
---|
303 | excold=$exc
|
---|
304 | exc=${results[$num+19]} # excrate in CU
|
---|
305 |
|
---|
306 | if [ "$onlyifhigher" = "yes" ]
|
---|
307 | then
|
---|
308 | higher=` echo " $exc > $oldexc " | bc `
|
---|
309 | if [ $higher -eq 1 ]
|
---|
310 | then
|
---|
311 | # keep old value
|
---|
312 | oldexc=$exc
|
---|
313 | fi
|
---|
314 | fi
|
---|
315 |
|
---|
316 | if [ "$bin" = "" ]
|
---|
317 | then
|
---|
318 | echo " ontime: "${results[$num+8]}" h" >> $logfile
|
---|
319 | fi
|
---|
320 |
|
---|
321 | # fast rise/decay trigger
|
---|
322 | if [ $triggertype -eq 3 ]
|
---|
323 | then
|
---|
324 | slopeprev=$slope
|
---|
325 | sigprev=$sig
|
---|
326 | if [ "$excold" = "0" ]
|
---|
327 | then
|
---|
328 | slope=0
|
---|
329 | else
|
---|
330 | slope=`echo " scale=1; ( $exc - $excold ) / ( $bin / 60. ) " | bc -l `
|
---|
331 | fi
|
---|
332 | get_query_minute_binning 60
|
---|
333 | #echo $query
|
---|
334 | results2=( `sendquery` )
|
---|
335 | # need to check last hour backward
|
---|
336 | if [ "${results2[19]}" = "" ]
|
---|
337 | then
|
---|
338 | slope60=0
|
---|
339 | sig60=0
|
---|
340 | else
|
---|
341 | sig60=${results2[18]} # significance 1h
|
---|
342 | if [ "${results2[27+19]}" = "" ]
|
---|
343 | then
|
---|
344 | # maybe treat this case differently
|
---|
345 | slope60=${results2[19]}
|
---|
346 | else
|
---|
347 | slope60=`echo " ${results2[19]} - ${results2[27+19]} " | bc -l` # ie /1h
|
---|
348 | fi
|
---|
349 | fi
|
---|
350 | #echo "exc "$exc" excold "$excold
|
---|
351 | #echo "slope "$slope" prev "$slopeprev" sig "$sig" prevsig "$sigprev" sig60 "$sig60" slope60 "$slope60
|
---|
352 | fi
|
---|
353 | # missing: probably one should check also 20 min binning (s example 20170103)
|
---|
354 | # missing: check on still available observation time
|
---|
355 |
|
---|
356 | echo " "$i" "${results[$num+2]}"-"${results[$num+3]}"["${results[$num+8]}"] "$exc" "$sig >> $logfile
|
---|
357 |
|
---|
358 | case $triggertype in
|
---|
359 | 1) #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
|
---|
360 | trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
|
---|
361 | limits=$exclimit"_"$siglimit
|
---|
362 | ;;
|
---|
363 | 2) #echo "magic 501: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
|
---|
364 | trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
|
---|
365 | limits=$exclimit"_"$siglimit
|
---|
366 | ;;
|
---|
367 | 3) #echo "magic fast rise/decay"
|
---|
368 | #trigger=`echo " $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit && $sig >= $siglimit " | bc -l`
|
---|
369 | trigger=`echo " ( ( $slope >= $slopelimit && $slopeprev >= $slopelimit && $slope60 >= $slopelimit ) || ( - $slope >= $slopelimit && - $slopeprev >= $slopelimit && - $slope60 >= $slopelimit ) ) && $sig >= $siglimit " | bc -l`
|
---|
370 | limits=$slopelimit"_"$siglimit
|
---|
371 | ;;
|
---|
372 | 5) #echo "std trigger: criteria ( $exc >= $exclimit && $sig >= $siglimit && $higher )"
|
---|
373 | trigger=`echo " $exc >= $exclimit && $sig >= $siglimit && $higher " | bc -l`
|
---|
374 | limits=$exclimit"_"$siglimit
|
---|
375 | ;;
|
---|
376 | *) echo $triggertype" not yet implemented" >> $logfile
|
---|
377 | ;;
|
---|
378 | esac
|
---|
379 |
|
---|
380 | # missing: maybe use $donetriggerfile for all cases - update of nightly case can be triggered differently
|
---|
381 | # missing: adapt triggerfilename for type 3 (slopelimit)
|
---|
382 | if [ $trigger -eq 1 ]
|
---|
383 | then
|
---|
384 | # do whatever to be done to trigger
|
---|
385 | # - send email/sms / call
|
---|
386 | # - create amon file
|
---|
387 | # - prepare email for alert
|
---|
388 | # - entry in DB
|
---|
389 |
|
---|
390 | if [ "$bin" = "" ]
|
---|
391 | then
|
---|
392 | # nightly file: simply overwrite, but do not send trigger again
|
---|
393 | # or check value if it's increasing?
|
---|
394 | # missing: get information from previous trigger + interpret
|
---|
395 | triggerfile=$flarealertspath"/"$night"-"$limits"-source"$sourcekey".trigger"$triggertype
|
---|
396 | else
|
---|
397 | # smaller binning: check if trigger is identical
|
---|
398 | triggerfile=$flarealertspath"/"$night"_"`printf %03d $runid`"-"$limits"-source"$sourcekey".trigger"$triggertype
|
---|
399 | fi
|
---|
400 | donetriggerfile=$triggerfile".done"
|
---|
401 | #ls $donetriggerfile
|
---|
402 |
|
---|
403 | # check if nightly flux increased
|
---|
404 | if [ "$bin" = "" ]
|
---|
405 | then
|
---|
406 | trigger2=1
|
---|
407 | changedfiles=( `ls $donetriggerfile"-changed-"* 2>/dev/null` )
|
---|
408 | if [ ${#changedfiles[@]} -gt 0 ]
|
---|
409 | then
|
---|
410 | oldnightlyexc=`grep "corr. cu: [0-9].[0-9] " ${changedfiles[@]} | grep -o -E ' [0-9].[0-9] ' | sort | tail -1`
|
---|
411 | trigger2=`echo " $exc > $oldnightlyexc " | bc -l`
|
---|
412 | echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")"
|
---|
413 | echo "trigger2: "$trigger2" (exc: "$exc", oldexc: "$oldnightlyexc")" >> $logfile
|
---|
414 | fi
|
---|
415 | fi
|
---|
416 |
|
---|
417 | # write new file only if old files do not agree
|
---|
418 | if [ -e $donetriggerfile ]
|
---|
419 | then
|
---|
420 | diff $donetriggerfile $triggerfile >/dev/null
|
---|
421 | checkstatus=`echo $?`
|
---|
422 | if [ $checkstatus -eq 0 ]
|
---|
423 | then
|
---|
424 | echo " alert already done "$donetriggerfile >> $logfile
|
---|
425 | i=`echo $i +1 | bc -l`
|
---|
426 | continue
|
---|
427 | fi
|
---|
428 | fi
|
---|
429 | if [ -e $triggerfile ]
|
---|
430 | then
|
---|
431 | mv $triggerfile $donetriggerfile
|
---|
432 | fi
|
---|
433 |
|
---|
434 | #echo $night"_"$runid" "$sourcekey" -> "$triggerfile
|
---|
435 | echo " writing "$triggerfile >> $logfile
|
---|
436 | touch $triggerfile
|
---|
437 | echo "Trigger found: " > $triggerfile
|
---|
438 | echo "-------------- " >> $triggerfile
|
---|
439 | echo " type: "$triggertype >> $triggerfile
|
---|
440 | echo " excess limit: "$exclimit" CU" >> $triggerfile
|
---|
441 | echo " significance limit: "$siglimit" sigma" >> $triggerfile
|
---|
442 | if [ "$bin" = "" ]
|
---|
443 | then
|
---|
444 | echo " nightly binning " >> $triggerfile
|
---|
445 | else
|
---|
446 | echo " binning: "$bin" min" >> $triggerfile
|
---|
447 | fi
|
---|
448 | echo "Summary of flare event: " >> $triggerfile
|
---|
449 | echo "----------------------- " >> $triggerfile
|
---|
450 | echo " source: "$sourcename >> $triggerfile
|
---|
451 | echo " night: "${results[$num+1]} >> $triggerfile
|
---|
452 | echo " runs: "${results[$num+2]}" - "${results[$num+3]} >> $triggerfile
|
---|
453 | echo " start: "${results[$num+4]}" "${results[$num+5]}" UTC" >> $triggerfile
|
---|
454 | echo " stop: "${results[$num+6]}" "${results[$num+7]}" UTC" >> $triggerfile
|
---|
455 | if [ "$bin" = "" ]
|
---|
456 | then
|
---|
457 | echo " ontime: "${results[$num+8]}" h" >> $triggerfile
|
---|
458 | else
|
---|
459 | echo " ontime: "${results[$num+8]}" min" >> $triggerfile
|
---|
460 | fi
|
---|
461 | #echo " ontime: "`echo "scale=1; ${results[$num+8]} / 60. " | bc -l`" min" #scale doesn't round properly
|
---|
462 | echo " signal: "${results[$num+9]}" evts" >> $triggerfile
|
---|
463 | echo " background: "${results[$num+10]}" evts" >> $triggerfile
|
---|
464 | echo " bgrate: "${results[$num+11]}" evts/h" >> $triggerfile
|
---|
465 | echo " exc: "${results[$num+12]}" +- "${results[$num+13]}" evts" >> $triggerfile
|
---|
466 | echo " excrate: "${results[$num+14]}" +- "${results[$num+15]}" evts/h" >> $triggerfile
|
---|
467 | echo " corr. excrate: "${results[$num+16]}" - "${results[$num+17]}" evts/h" >> $triggerfile
|
---|
468 | echo " significance: "${results[$num+18]}" sigma" >> $triggerfile
|
---|
469 | echo " cu: "${results[$num+19]}" +- "${results[$num+20]}" CU" >> $triggerfile
|
---|
470 | echo " corr. cu: "${results[$num+21]}" +- "${results[$num+22]}" CU" >> $triggerfile
|
---|
471 | echo " zd: "${results[$num+23]}" - "${results[$num+24]}" degree" >> $triggerfile
|
---|
472 | echo " th: "${results[$num+25]}" - "${results[$num+26]}" DAC counts" >> $triggerfile
|
---|
473 | # additional information fast rise/decay trigger
|
---|
474 | if [ $triggertype -eq 3 ]
|
---|
475 | then
|
---|
476 | echo "Flux doubling/halfing times: " >> $triggerfile
|
---|
477 | echo "---------------------------- " >> $triggerfile
|
---|
478 | echo " excess: "$exc >> $triggerfile
|
---|
479 | echo " excess old: "$excold >> $triggerfile
|
---|
480 | echo " significance: "$sig >> $triggerfile
|
---|
481 | echo " significance old: "$sigprev >> $triggerfile
|
---|
482 | echo " slope: "$excold >> $triggerfile
|
---|
483 | echo " slope old: "$slopeold >> $triggerfile
|
---|
484 | echo " excess 60 min: "${results2[19]} >> $triggerfile
|
---|
485 | echo " excess 60 min old: "${results2[27+19]} >> $triggerfile
|
---|
486 | echo " slope 60 min: "$slope60 >> $triggerfile
|
---|
487 | echo " significance 60 min: "$sig60 >> $triggerfile
|
---|
488 | fi
|
---|
489 |
|
---|
490 | if [ -e $donetriggerfile ]
|
---|
491 | then
|
---|
492 | diff $donetriggerfile $triggerfile >/dev/null
|
---|
493 | checkstatus=`echo $?`
|
---|
494 | if [ $checkstatus -gt 0 ]
|
---|
495 | then
|
---|
496 | # keep history of non-sent triggers
|
---|
497 | donetriggerfile2=$donetriggerfile"-changed-"`date +%Y%m%d%H%M%S`
|
---|
498 | cp $donetriggerfile $donetriggerfile2
|
---|
499 | fi
|
---|
500 | fi
|
---|
501 | # missing: get summary of whole observation
|
---|
502 |
|
---|
503 | # send email only of $donetriggerfile doesn't exists
|
---|
504 | if ! [ -e $donetriggerfile ]
|
---|
505 | then
|
---|
506 | query="INSERT FlareAlerts.FlareTriggers SET fTriggerInserted=Now(), fNight="$night", fRunID="$runid", fTriggerType="$triggertype
|
---|
507 | if [ "$bin" = "" ]
|
---|
508 | then
|
---|
509 | query=$query", fBinning=NULL"
|
---|
510 | else
|
---|
511 | query=$query", fBinning="$bin
|
---|
512 | fi
|
---|
513 | echo $query
|
---|
514 | sendquery >> $logfile
|
---|
515 |
|
---|
516 | # AMON case: create VOEvent-File
|
---|
517 | if [ $triggertype -eq 5 ]
|
---|
518 | then
|
---|
519 | voeventfile=$voeventpath"/"`basename $triggerfile`".xml"
|
---|
520 | # missing: check for archive file
|
---|
521 | # if exist -> do revision
|
---|
522 | echo "creating "$voeventfile
|
---|
523 | echo "Creating "$voeventfile >> $logfile
|
---|
524 | #print_voevent_file
|
---|
525 | print_voevent_file > $voeventfile
|
---|
526 | # in amon-case no email needs to be sent
|
---|
527 | else
|
---|
528 | if [ "$bin" = "" ] && [ $trigger2 -eq 0 ]
|
---|
529 | then
|
---|
530 | continue
|
---|
531 | fi
|
---|
532 | echo "sending["$triggertype"] "$triggerfile
|
---|
533 | echo "sending["$triggertype"] "$triggerfile >> $logfile
|
---|
534 | cat $triggerfile | mail -s 'test flare alert ' -b $emailfrom -r $emailfrom $emailto
|
---|
535 | #cat $triggerfile | mail -s "test flare alert for $sourcename " $emailto
|
---|
536 | # that's also the cases for making a call
|
---|
537 | # fill DB for shifthelper
|
---|
538 | fi
|
---|
539 | fi
|
---|
540 | fi
|
---|
541 |
|
---|
542 | # counter
|
---|
543 | i=`echo $i +1 | bc -l`
|
---|
544 | done
|
---|
545 |
|
---|
546 | echo " found "$i" data point(s)." >> $logfile
|
---|
547 | echo "" >> $logfile
|
---|
548 | }
|
---|
549 |
|
---|
550 | get_average_flux()
|
---|
551 | {
|
---|
552 | # query average flux from DB
|
---|
553 | query="SELECT ROUND(SUM(fNumExcEvts)/SUM("$ontime")*3600,1) as excrate, "
|
---|
554 | query=$query"ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontime")*3600, 1) AS excerr, "
|
---|
555 | query=$query"ROUND(SUM("$ontime")/3600.,1) AS ontime "
|
---|
556 | query=$query"FROM AnalysisResultsRunLP "
|
---|
557 | query=$query"LEFT JOIN RunInfo USING (fNight, fRunID) "
|
---|
558 | if [ "$1" = "" ]
|
---|
559 | then
|
---|
560 | query=$query"WHERE fSourceKey="$sourcekey" AND NOT ISNULL(fNumExcEvts) "
|
---|
561 | else
|
---|
562 | query=$query"WHERE fSourceKey="$sourcekey" AND fNight BETWEEN "$1" AND "$2" AND NOT ISNULL(fNumExcEvts) "
|
---|
563 | fi
|
---|
564 | query=$query"GROUP BY fSourceKey "
|
---|
565 | sendquery
|
---|
566 | # missing - get in CU to correct for fluctuations of CU
|
---|
567 | }
|
---|
568 |
|
---|
569 |
|
---|
570 |
|
---|
571 | # main part of the script
|
---|
572 |
|
---|
573 | # missing: check if twistd client is running
|
---|
574 | # + check if there are remaining files in the to-send-folder of amon
|
---|
575 | # -> send email if amon connection has problem
|
---|
576 |
|
---|
577 |
|
---|
578 | for sourcekey in ${sourcekeys[@]}
|
---|
579 | do
|
---|
580 | query="SELECT fSourceName FROM Source WHERE fSourceKey="$sourcekey
|
---|
581 | sourcename=`sendquery` #do not combine this with other source info as sourcename can have spaces
|
---|
582 |
|
---|
583 | # todo: what about data check ?
|
---|
584 | # should avg include current night?
|
---|
585 | total=( `get_average_flux` )
|
---|
586 | month=( `get_average_flux \`date -d $night' - 1 MONTH' +%Y%m%d\` $night` )
|
---|
587 | year=( `get_average_flux \`date -d $night' - 1 YEAR' +%Y%m%d\` $night` )
|
---|
588 | avgflux=${total[0]}
|
---|
589 | avgfluxmonth=${month[0]}
|
---|
590 | avgfluxyear=${year[0]}
|
---|
591 | error=${total[1]}
|
---|
592 | errormonth=${month[1]}
|
---|
593 | erroryear=${year[1]}
|
---|
594 |
|
---|
595 | # getting some information on the source
|
---|
596 | query="SELECT fDeclination, fRightAscension, fRightAscension/24.*15 FROM Source WHERE fSourceKey="$sourcekey
|
---|
597 | sourceinfo=( `sendquery` )
|
---|
598 |
|
---|
599 | # ignore Crab
|
---|
600 | if [ $sourcekey -eq 5 ]
|
---|
601 | then
|
---|
602 | continue
|
---|
603 | fi
|
---|
604 | printprocesslog "INFO Evaluation for $sourcename ... "
|
---|
605 | echo "Evaluation for $sourcename ... " >> $logfile
|
---|
606 | echo " average fluxes: "$avgfluxmonth"+-"$errormonth" evts/h (last month) "$avgfluxyear"+-"$erroryear" evts/h (last year) "$avgflux"+-"$error" evts/h (all)" >> $logfile
|
---|
607 |
|
---|
608 | # missing: get limits from DB (structure needs to be defined)
|
---|
609 |
|
---|
610 | # triggers in the frame of the MoU in the gamma-ray community
|
---|
611 | triggertype=1
|
---|
612 | # limits
|
---|
613 | siglimit=3.0 # sigma
|
---|
614 | #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ]
|
---|
615 | if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 5 ]
|
---|
616 | then
|
---|
617 | exclimit=3.0 # CU
|
---|
618 | else
|
---|
619 | exclimit=0.5 # CU
|
---|
620 | fi
|
---|
621 | # only if rate goes even higher, we have to react
|
---|
622 | onlyifhigher="yes"
|
---|
623 | higher=1
|
---|
624 |
|
---|
625 | printprocesslog "INFO checking for [General gamma-ray MoU]" >> $logfile
|
---|
626 | echo "[General gamma-ray MoU]" >> $logfile
|
---|
627 | echo " nightly binning..." >> $logfile
|
---|
628 | # checking nightly binning
|
---|
629 | bin=
|
---|
630 | get_query_nightly_binning
|
---|
631 | results=( `sendquery` )
|
---|
632 | evaluate_result
|
---|
633 |
|
---|
634 | # 20 min binning
|
---|
635 | bin=20
|
---|
636 | echo " "$bin" min binning..." >> $logfile
|
---|
637 | get_query_minute_binning $bin
|
---|
638 | results=( `sendquery` )
|
---|
639 | evaluate_result
|
---|
640 |
|
---|
641 | # triggers to MAGIC
|
---|
642 | # Mrk 501 proposal
|
---|
643 | triggertype=2
|
---|
644 | if [ $sourcekey -eq 2 ]
|
---|
645 | then
|
---|
646 | siglimit=3.0
|
---|
647 | exclimit=2.0 # cu
|
---|
648 | printprocesslog "INFO checking for [Trigger to MAGIC 501 proposal]" >> $logfile
|
---|
649 | echo "[Trigger to MAGIC 501 proposal]" >> $logfile
|
---|
650 | # checking nightly binning
|
---|
651 | bin=
|
---|
652 | echo " nightly binning..." >> $logfile
|
---|
653 | get_query_nightly_binning
|
---|
654 | results=( `sendquery` )
|
---|
655 | evaluate_result
|
---|
656 | # 20 min binning
|
---|
657 | bin=20
|
---|
658 | echo " "$bin" min binning..." >> $logfile
|
---|
659 | get_query_minute_binning $bin
|
---|
660 | results=( `sendquery` )
|
---|
661 | evaluate_result
|
---|
662 | fi
|
---|
663 | # Mother of ToO - fast rise/decay
|
---|
664 | # sources: Mrk 421, Mrk 501, 2344, 1959
|
---|
665 | triggertype=3
|
---|
666 | #if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ] || [ $sourcekey -eq 5 ] # for testing
|
---|
667 | if [ $sourcekey -eq 1 ] || [ $sourcekey -eq 2 ] || [ $sourcekey -eq 3 ] || [ $sourcekey -eq 7 ]
|
---|
668 | then
|
---|
669 | printprocesslog "INFO checking for [Trigger to MAGIC - fast rise/decay]" >> $logfile
|
---|
670 | echo "[Trigger to MAGIC - fast rise/decay]" >> $logfile
|
---|
671 | ## keep thresholds low (or do not use in evaluation)
|
---|
672 | #siglimit=2.0
|
---|
673 | #exclimit=0.5
|
---|
674 | # limits in slope
|
---|
675 | slopelimit=1.0 # 1CU/h
|
---|
676 | siglimit=3.0 # 1 sigma in 1 hour
|
---|
677 | # binning
|
---|
678 | bin=30
|
---|
679 | echo " "$bin" min binning..." >> $logfile
|
---|
680 | get_query_minute_binning $bin
|
---|
681 | results=( `sendquery` )
|
---|
682 | evaluate_result
|
---|
683 | fi
|
---|
684 |
|
---|
685 | # X-ray ToO
|
---|
686 | triggertype=4
|
---|
687 | # to be added
|
---|
688 |
|
---|
689 | # AMON - automatic triggers using VOEvent files
|
---|
690 | triggertype=5
|
---|
691 | printprocesslog "INFO checking for [Trigger to AMON]" >> $logfile
|
---|
692 | echo "[Trigger to AMON]" >> $logfile
|
---|
693 | echo " details still to be defined" >> $logfile
|
---|
694 | # missing: trigger limits and binning still to be defined
|
---|
695 | # x times above average + significance limit ?
|
---|
696 | # sub-threshold?
|
---|
697 | # FP-rate to be calculated
|
---|
698 | # use for the moment 0.5 CU and 3 sigma
|
---|
699 | siglimit=3.0
|
---|
700 | exclimit=0.5
|
---|
701 | onlyifhigher="no"
|
---|
702 | # checking nightly binning
|
---|
703 | bin=
|
---|
704 | echo " nightly binning..." >> $logfile
|
---|
705 | get_query_nightly_binning
|
---|
706 | results=( `sendquery` )
|
---|
707 | evaluate_result
|
---|
708 | bin=20
|
---|
709 | echo " "$bin" min binning..." >> $logfile
|
---|
710 | get_query_minute_binning $bin
|
---|
711 | results=( `sendquery` )
|
---|
712 | evaluate_result
|
---|
713 |
|
---|
714 | echo "" >> $logfile
|
---|
715 | echo "" >> $logfile
|
---|
716 | done
|
---|
717 |
|
---|
718 | finish
|
---|
719 |
|
---|
720 | # for archival testing:
|
---|
721 | for (( i=0; i < 100 ; i++))
|
---|
722 | do
|
---|
723 | date=`date --date="-${i}days" +%Y%m%d`
|
---|
724 | /home/fact/SW.automatic.processing/DataCheck/QuickLook/FlareAlerts.sh $date
|
---|
725 | done
|
---|
726 |
|
---|
727 |
|
---|