| 1 | #!/bin/bash
|
|---|
| 2 | #
|
|---|
| 3 |
|
|---|
| 4 | source `dirname $0`/../Sourcefile.sh
|
|---|
| 5 | printprocesslog "INFO starting $0"
|
|---|
| 6 |
|
|---|
| 7 | echo "Sent data."
|
|---|
| 8 |
|
|---|
| 9 | start=$1
|
|---|
| 10 | stop=$2
|
|---|
| 11 | source=$3
|
|---|
| 12 | bin=$4
|
|---|
| 13 | email=$5
|
|---|
| 14 | table=$6
|
|---|
| 15 | time=$7
|
|---|
| 16 | expert=$8
|
|---|
| 17 |
|
|---|
| 18 | query="SELECT fSourceName FROM Source WHERE fSourceKey="$source
|
|---|
| 19 | #echo $query
|
|---|
| 20 | sourcename=`sendquery`
|
|---|
| 21 |
|
|---|
| 22 | function printinfo()
|
|---|
| 23 | {
|
|---|
| 24 | echo "# "
|
|---|
| 25 | echo "# Please cite the FACT design paper and the QLA webpage when using these data."
|
|---|
| 26 | echo "# FACT design paper: H. Anderhub et al. JINST 8 (2013) P6008 "
|
|---|
| 27 | echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
|
|---|
| 28 | echo "# QLA webpage: http://www.fact-project.org/monitoring"
|
|---|
| 29 | echo "# If you intent to use the data, please let us know for reference. "
|
|---|
| 30 | echo "# "
|
|---|
| 31 | echo "# Remarks:"
|
|---|
| 32 | echo "# * These are the results of a fast quick look analyis "
|
|---|
| 33 | echo "# on site, i.e. they are preliminary. "
|
|---|
| 34 | echo "# * The quick look analysis includes all data, "
|
|---|
| 35 | echo "# i.e. no data selection done."
|
|---|
| 36 | echo "# * The given values are not fluxes but excess rates "
|
|---|
| 37 | echo "# (number of excess events per effective ontime), "
|
|---|
| 38 | echo "# i.e. there is a dependence on trigger threshold and "
|
|---|
| 39 | echo "# zenith distance of the observation (with the current "
|
|---|
| 40 | echo "# analysis for zenith distance > 40 degree and trigger "
|
|---|
| 41 | echo "# threshold > 500 DAC counts)."
|
|---|
| 42 | echo "# * Nights with less than 20 minutes of data are neglected "
|
|---|
| 43 | echo "# for nightly binning. "
|
|---|
| 44 | echo "# * The QLA results are not reprocessed when a new software "
|
|---|
| 45 | echo "# version is introduced. "
|
|---|
| 46 | echo "# * In case, you need further details about the data or a"
|
|---|
| 47 | echo "# different binning, please do not hesitate to contact us."
|
|---|
| 48 | echo "# * The QLA contains all data since 12.12.2012. "
|
|---|
| 49 | echo "# For older data, please contact us. "
|
|---|
| 50 | echo "# "
|
|---|
| 51 | echo "# Contact: Daniela Dorner dorner@astro.uni-wuerzburg.de "
|
|---|
| 52 | echo "# "
|
|---|
| 53 | echo "# This file was created at "`date`
|
|---|
| 54 | echo "# Source: "$sourcename
|
|---|
| 55 | echo "# Timerange: "$start" - "$stop
|
|---|
| 56 | if [ $bin -lt 0 ]
|
|---|
| 57 | then
|
|---|
| 58 | echo "# Binning: nightly "
|
|---|
| 59 | else
|
|---|
| 60 | echo "# Binning: "$bin" min "
|
|---|
| 61 | fi
|
|---|
| 62 | echo "# "
|
|---|
| 63 | #echo "# start("$time") stop("$time") excess-rate(evts/h) error-excess-rate(evts/h) ontime(min) significance[if Nsig>10 and Nbg>10 (or Nexc>10)] "
|
|---|
| 64 | echo "# start("$time") stop("$time") excess-rate(evts/h) error-excess-rate(evts/h) "
|
|---|
| 65 | getdbsetup
|
|---|
| 66 | printprocesslog "DEBUG sendquery QUERY: "$query
|
|---|
| 67 | mysql $sqlpwfile -s -u $us $pw --host=$ho $db -e " $query "
|
|---|
| 68 | }
|
|---|
| 69 |
|
|---|
| 70 | ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
|
|---|
| 71 | ontime2=" fOnTimeAfterCuts "
|
|---|
| 72 | ontimeif=" IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn) "
|
|---|
| 73 | from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
|
|---|
| 74 | # time range and source
|
|---|
| 75 | where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$start" AND "$stop
|
|---|
| 76 | # some sanity checks
|
|---|
| 77 | where=$where" AND fRunTypeKey=1 AND NOT ISNULL(fNumSigEvts) AND NOT ISNULL(fNumBgEvts) AND NOT fRunStart='0000-00-00 00:00:00' AND NOT fRunStop='0000-00-00 00:00:00' "
|
|---|
| 78 |
|
|---|
| 79 | case $time in
|
|---|
| 80 | mjd) timeselect="Mjd(Min(fRunStart)) AS start, Mjd(MAX(fRunStop)) AS stop, "
|
|---|
| 81 | timeselect2=" Mjd(MIN(o.start)) AS start, Mjd(MAX(o.stop)) AS stop, "
|
|---|
| 82 | ;;
|
|---|
| 83 | unix) timeselect="Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')) AS start, Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) AS stop, "
|
|---|
| 84 | timeselect2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start, Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop, "
|
|---|
| 85 | ;;
|
|---|
| 86 | *) timeselect="MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
|
|---|
| 87 | timeselect2=" MIN(o.start) AS start, MAX(o.stop) as stop, "
|
|---|
| 88 | ;;
|
|---|
| 89 | esac
|
|---|
| 90 |
|
|---|
| 91 | # todo:
|
|---|
| 92 | # implement login
|
|---|
| 93 | # implement for internal use
|
|---|
| 94 | # table
|
|---|
| 95 | # more binnings (insert box for minutes? period, 10d?, weekly?)
|
|---|
| 96 | # zd limit
|
|---|
| 97 | # threshold limit
|
|---|
| 98 | # datacheck
|
|---|
| 99 |
|
|---|
| 100 | if [ $bin -eq -1 ]
|
|---|
| 101 | then
|
|---|
| 102 | #query="SELECT Min(fRunStart), Max(fRunStop), Sum(fNumExcEvts)/Sum(fOnTimeAfterCuts)*3600, if ((Sum(fNumSigEvts) + 0.2*0.2*Sum(fNumBgEvts))<0, 0, Sqrt(Sum(fNumSigEvts) + 0.2*0.2*Sum(fNumBgEvts))) FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) WHERE fSourceKey="$source" AND fNight BETWEEN "$start" AND "$stop" GROUP BY fNight HAVING SUM(fOnTimeAfterCuts)*60>20"
|
|---|
| 103 | query="SELECT "$timeselect" SUM(fNumExcEvts)/SUM("$ontimeif")*3600 AS excrate, "
|
|---|
| 104 | query=$query" ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontimeif")*3600 AS excerr "
|
|---|
| 105 | #query=$query" IF (SUM(fNumSigEvts)>10 AND SUM(fNumBgEvts)>10, LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)), IF (SUM(fNumSigEvts)-SUM(fNumBgEvts)>10, concat('(', LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)), ')'), '-')) AS significance "
|
|---|
| 106 | #query=$query" Avg(fThresholdMedian) as th "
|
|---|
| 107 | query=$query$from$where" GROUP BY fNight HAVING SUM("$ontimeif")/60>20 ORDER BY fNight "
|
|---|
| 108 | else
|
|---|
| 109 | # query to get excess rate and significance from DB
|
|---|
| 110 | query="SELECT "$timeselect2" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 2) AS excrate, "
|
|---|
| 111 | query=$query" ROUND(ExcErr(SUM(o.sigevts),SUM(o.bgevts))/SUM(o.ot)*3600,2) AS err "
|
|---|
| 112 | #query=$query" ROUND(SUM(o.ot)/60.,1) AS 'ontime [min]' "
|
|---|
| 113 | #query=$query" IF (SUM(o.sigevts)>10 AND SUM(o.bgevts)>10, LiMa(SUM(o.sigevts), SUM(o.bgevts)), IF (SUM(o.sigevts)-SUM(o.bgevts)>10, CONCAT('(', LiMa(SUM(o.sigevts), SUM(o.bgevts)), ')'), '-')) AS significance "
|
|---|
| 114 | query=$query" FROM (SELECT fNight, @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "
|
|---|
| 115 | query=$query" IF (@night=fNight AND FLOOR((@os+@ot)/20./60.)<1, @bl, @bl := @bl + 1) AS block, "
|
|---|
| 116 | query=$query" IF (@night=fNight AND FLOOR((@os+@ot)/20./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
|
|---|
| 117 | query=$query$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
|
|---|
| 118 | query=$query$where" ORDER BY fRunStart) o GROUP BY block ORDER BY start"
|
|---|
| 119 | fi
|
|---|
| 120 | #echo $query
|
|---|
| 121 |
|
|---|
| 122 | #printinfo | mail -a "From: dorner@astro.uni-wuerzburg.de" -s 'test downloading data ' -b dorner@astro.uni-wuerzburg.de $email
|
|---|
| 123 | printinfo | mail -s 'FACT data (test download)' -b dorner@astro.uni-wuerzburg.de -r dorner@astro.uni-wuerzburg.de $email
|
|---|
| 124 | #printinfo
|
|---|
| 125 |
|
|---|
| 126 | finish
|
|---|
| 127 |
|
|---|
| 128 | # new query:
|
|---|
| 129 | # select o.night, Min(o.start), Max(o.stop), Sum(o.sigevts), Sum(o.bgevts), Round(Sum(o.ot)/60.,1) FROM (Select fNight, fRunID, fOnTimeAfterCuts as ot, fRunStart as start, fRunStop as stop, fNumSigEvts as sigevts, fNumBgEvts as bgevts, if (@night=fNight AND floor((@os+fOnTimeAfterCuts)/20./60.)<1, @bl, @bl := @bl + 1) as block, if (@night=fNight AND floor((@os+fOnTimeAfterCuts)/20./60.)<1, @os:=@os + fOnTimeAfterCuts, @os := fOnTimeAfterCuts) as os, @os/20./60. as a, floor(@os/20./60.) as b, @night :=fNight as night from RunInfo left join AnalysisResultsRunLP using(fNight,fRunID) CROSS JOIN (SELECT @night :=0, @os :=0, @bl:=0) PARAMS where fNight between 20130414 and 20130415 and fRunTypeKey=1 and fSourceKey=1) o group by block;
|
|---|
| 130 |
|
|---|
| 131 | # input
|
|---|
| 132 | # - binning
|
|---|
| 133 | # - source
|
|---|
| 134 | # - email address
|
|---|
| 135 | # - night range
|
|---|
| 136 |
|
|---|
| 137 | # binning
|
|---|
| 138 | bin=20
|
|---|
| 139 |
|
|---|
| 140 | # get date
|
|---|
| 141 | # add funtion
|
|---|
| 142 | night=20160102
|
|---|
| 143 |
|
|---|
| 144 | # get sources for current night from DB (RunInfo)
|
|---|
| 145 | sourcekey=1
|
|---|
| 146 |
|
|---|
| 147 | # get limits from DB
|
|---|
| 148 | siglimit=3.0
|
|---|
| 149 | exclimit=20
|
|---|
| 150 |
|
|---|
| 151 | # query to get excess rate and significance from DB
|
|---|
| 152 | query="SELECT o.* FROM"
|
|---|
| 153 | query=$query" (SELECT fRunID, IF (@os+fOnTimeAfterCuts/60.>"$bin", @sig := fNumSigEvts, @sig := @sig + fNumSigEvts) AS sigevts, "
|
|---|
| 154 | query=$query" IF (@os+fOnTimeAfterCuts/60.>"$bin", @bg := fNumBgEvts, @bg := @bg + fNumBgEvts) AS bgevts, "
|
|---|
| 155 | query=$query" IF (@os+fOnTimeAfterCuts/60.>"$bin", @exc := fNumExcEvts, @exc := @exc + fNumExcEvts) AS excevts, "
|
|---|
| 156 | query=$query" IF (@os+fOnTimeAfterCuts/60.>"$bin", @os := fOnTimeAfterCuts/60., @os := @os + fOnTimeAfterCuts/60.) AS ontime, "
|
|---|
| 157 | query=$query" @exc/@os*60 AS excrate, "
|
|---|
| 158 | query=$query" IF (@sig>0 AND @bg*5>0 AND (@sig*Log(@sig/(@sig+@bg*5)*1.2/0.2)+@bg*5*Log(@bg*5/(@sig+@bg*5)*1.2))> 0, Sqrt((@sig*Log(@sig/(@sig+@bg*5)*1.2/0.2)+@bg*5*Log(@bg*5/(@sig+@bg*5)*1.2))*2), 0) AS sig "
|
|---|
| 159 | query=$query" FROM AnalysisResultsRunLP O LEFT JOIN RunInfo USING (fNight, fRunID) "
|
|---|
| 160 | query=$query" CROSS JOIN (SELECT @os :=0, @sig :=0, @bg := 0, @exc :=0, @rate :=0) PARAMS "
|
|---|
| 161 | query=$query" WHERE fNight="$night" AND fSourceKey="$sourcekey") o HAVING ontime > "$bin"*0.9"
|
|---|
| 162 | results=( `sendquery` )
|
|---|
| 163 | #echo ${results[@]}
|
|---|
| 164 |
|
|---|
| 165 | i=0
|
|---|
| 166 | while [ 0 -lt 1 ]
|
|---|
| 167 | do
|
|---|
| 168 | num=`echo "$i * 7" | bc -l`
|
|---|
| 169 | if [ "${results[$num]}" = "" ]
|
|---|
| 170 | then
|
|---|
| 171 | break
|
|---|
| 172 | fi
|
|---|
| 173 | runid=${results[$num]}
|
|---|
| 174 | sig=${results[$num+6]}
|
|---|
| 175 | exc=${results[$num+5]}
|
|---|
| 176 |
|
|---|
| 177 | trigger=`echo " $exc > $exclimit && $sig > $siglimit " | bc -l`
|
|---|
| 178 |
|
|---|
| 179 | # echo $i" "$num" "$exc" "$sig
|
|---|
| 180 | if [ $trigger -eq 1 ]
|
|---|
| 181 | then
|
|---|
| 182 | # do whatever to be done to trigger
|
|---|
| 183 | # - send email/sms / call
|
|---|
| 184 | # - create amon file
|
|---|
| 185 | # - prepare email for alert
|
|---|
| 186 | echo $runid" "$exc" "$sig" -> "$trigger
|
|---|
| 187 | triggerfile="/home/fact/flare.alerts/"$night"_"$runid"_"$sourcekey".trigger"
|
|---|
| 188 | touch $triggerfile
|
|---|
| 189 | fi
|
|---|
| 190 | # counter
|
|---|
| 191 | i=`echo $i +1 | bc -l`
|
|---|
| 192 | done
|
|---|
| 193 |
|
|---|
| 194 | finish
|
|---|
| 195 |
|
|---|
| 196 |
|
|---|
| 197 | # old query:
|
|---|
| 198 | # #query="SELECT "$timeselect2", Round(o.excrate,2), Round(ExcErr(o.sigevts, o.bgevts),2), IF (o.sigevts>10 AND o.bgevts>10, LiMa(o.sigevts, o.bgevts), '-'), Round(o.ontime,1) as ontime FROM"
|
|---|
| 199 | # query="SELECT "$timeselect2" AS time, ROUND((o.sigevts-o.bgevts)/o.ontime*60,2) AS excrate, ROUND(ExcErr(o.sigevts, o.bgevts)/o.ontime*60,2) AS excerr, ROUND(o.ontime,1) AS ontime, IF (o.sigevts>10 AND o.bgevts>10, LiMa(o.sigevts, o.bgevts), IF (o.sigevts-o.bgevts>10, CONCAT('(', LiMa(o.sigevts, o.bgevts), ')'), '-')) AS significance FROM"
|
|---|
| 200 | # query=$query" (SELECT IF (@os+fOnTimeAfterCuts/60.>20, @start:=fRunStart, IF (@os=0, @start:=fRunStart, @start)) as start, fRunStop as stop, IF (@os+fOnTimeAfterCuts/60.>"$bin", @sig := fNumSigEvts, @sig := @sig + fNumSigEvts) AS sigevts, "
|
|---|
| 201 | # #query=$query" (SELECT fRunID, IF (@os+fOnTimeAfterCuts/60.>"$bin", @sig := fNumSigEvts, @sig := @sig + fNumSigEvts) AS sigevts, "
|
|---|
| 202 | # query=$query" IF (@os+fOnTimeAfterCuts/60.>"$bin", @bg := fNumBgEvts, @bg := @bg + fNumBgEvts) AS bgevts, "
|
|---|
| 203 | # query=$query" IF (@os+fOnTimeAfterCuts/60.>"$bin", @exc := fNumExcEvts, @exc := @exc + fNumExcEvts) AS excevts, "
|
|---|
| 204 | # query=$query" IF (@os+fOnTimeAfterCuts/60.>"$bin", @os := fOnTimeAfterCuts/60., @os := @os + fOnTimeAfterCuts/60.) AS ontime "
|
|---|
| 205 | # #query=$query" @exc/@os*60 AS excrate, "
|
|---|
| 206 | # #query=$query" ExcErr(@sig, @bg) AS excerr, "
|
|---|
| 207 | # #query=$query" IF (@sig>0 AND @bg*5>0 AND (@sig*Log(@sig/(@sig+@bg*5)*1.2/0.2)+@bg*5*Log(@bg*5/(@sig+@bg*5)*1.2))> 0, Sqrt((@sig*Log(@sig/(@sig+@bg*5)*1.2/0.2)+@bg*5*Log(@bg*5/(@sig+@bg*5)*1.2))*2), 0) AS sig "
|
|---|
| 208 | # #query=$query" LiMa(@sig,@bg) AS signif "
|
|---|
| 209 | # query=$query" FROM AnalysisResultsRunLP O LEFT JOIN RunInfo USING (fNight, fRunID) "
|
|---|
| 210 | # query=$query" CROSS JOIN (SELECT @os :=0, @sig :=0, @bg := 0, @exc :=0, @rate :=0, @start :=0) PARAMS "
|
|---|
| 211 | # query=$query" WHERE fNight between "$start" AND "$stop" AND fSourceKey="$source" ORDER BY fNight, fRunID) o HAVING ontime > "$bin"*0.75"
|
|---|
| 212 | #
|
|---|
| 213 |
|
|---|
