| 1 | #!/bin/bash
|
|---|
| 2 | #
|
|---|
| 3 |
|
|---|
| 4 | source `dirname $0`/../Sourcefile.sh
|
|---|
| 5 | printprocesslog "INFO starting $0"
|
|---|
| 6 |
|
|---|
| 7 | echo "Sent data"
|
|---|
| 8 |
|
|---|
| 9 | start=$1
|
|---|
| 10 | stop=$2
|
|---|
| 11 | source=$3
|
|---|
| 12 | bin=$4
|
|---|
| 13 | email=$5
|
|---|
| 14 | table=$6
|
|---|
| 15 | time=$7
|
|---|
| 16 | expert=$8
|
|---|
| 17 | dch=$9
|
|---|
| 18 | zdcut=${10}
|
|---|
| 19 | thcut=${11}
|
|---|
| 20 | dark=${12}
|
|---|
| 21 | dust=${13}
|
|---|
| 22 |
|
|---|
| 23 |
|
|---|
| 24 | query="SELECT fSourceName FROM Source WHERE fSourceKey="$source
|
|---|
| 25 | sourcename=`sendquery`
|
|---|
| 26 |
|
|---|
| 27 | function printinfo()
|
|---|
| 28 | {
|
|---|
| 29 | echo "# "
|
|---|
| 30 | echo "# Please cite the FACT design paper and the QLA webpage when using these data."
|
|---|
| 31 | echo "# FACT design paper: H. Anderhub et al. JINST 8 (2013) P6008 "
|
|---|
| 32 | echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
|
|---|
| 33 | echo "# QLA webpage: http://www.fact-project.org/monitoring"
|
|---|
| 34 | echo "# If you intent to use the data, please let us know for reference. "
|
|---|
| 35 | echo "# "
|
|---|
| 36 | echo "# Remarks:"
|
|---|
| 37 | echo "# * These are the results of a fast quick look analyis "
|
|---|
| 38 | echo "# on site, i.e. they are preliminary. "
|
|---|
| 39 | echo "# * The quick look analysis includes all data, "
|
|---|
| 40 | echo "# i.e. no data selection done."
|
|---|
| 41 | echo "# * The given values are not fluxes but excess rates "
|
|---|
| 42 | echo "# (number of excess events per effective ontime), "
|
|---|
| 43 | echo "# i.e. there is a dependence on trigger threshold and "
|
|---|
| 44 | echo "# zenith distance of the observation (with the current "
|
|---|
| 45 | echo "# analysis for zenith distance > 40 degree and trigger "
|
|---|
| 46 | echo "# threshold > 500 DAC counts)."
|
|---|
| 47 | echo "# * Nights with less than 20 minutes of data are neglected "
|
|---|
| 48 | echo "# for nightly binning. "
|
|---|
| 49 | echo "# * The QLA results are not reprocessed when a new software "
|
|---|
| 50 | echo "# version is introduced. "
|
|---|
| 51 | echo "# * In case, you need further details about the data or a"
|
|---|
| 52 | echo "# different binning, please do not hesitate to contact us."
|
|---|
| 53 | echo "# * The QLA contains all data since 12.12.2012. "
|
|---|
| 54 | echo "# For older data, please contact us. "
|
|---|
| 55 | echo "# "
|
|---|
| 56 | echo "# Contact: Daniela Dorner dorner@astro.uni-wuerzburg.de "
|
|---|
| 57 | echo "# "
|
|---|
| 58 | echo "# This file was created at "`date`
|
|---|
| 59 | echo "# Source: "$sourcename
|
|---|
| 60 | echo "# Timerange: "$start" - "$stop
|
|---|
| 61 | echo "# Table: "$table
|
|---|
| 62 | echo "# Zdcut: "$zdcut
|
|---|
| 63 | echo "# thcut: "$thcut
|
|---|
| 64 | echo "# moon zd: "$dark
|
|---|
| 65 | echo "# dust: "$dust
|
|---|
| 66 | if [ $dch -eq 0 ]
|
|---|
| 67 | then
|
|---|
| 68 | echo "# no datacheck"
|
|---|
| 69 | else
|
|---|
| 70 | echo "# some rough automatic datacheck applied"
|
|---|
| 71 | fi
|
|---|
| 72 | if [ $bin -lt 0 ]
|
|---|
| 73 | then
|
|---|
| 74 | echo "# Binning: nightly "
|
|---|
| 75 | else
|
|---|
| 76 | echo "# Binning: "$bin" min "
|
|---|
| 77 | fi
|
|---|
| 78 | echo "# "
|
|---|
| 79 | echo "# start("$time") stop("$time") excess-rate(evts/h) error-excess-rate(evts/h) ontime(min) significance[if Nsig>10 and Nbg>10 (or Nexc>10)] significance zdmean thmean sigrate[evts/h] bgrate[evts/h] corr-factor bg-corr-factor"
|
|---|
| 80 | #echo "# start("$time") stop("$time") excess-rate(evts/h) error-excess-rate(evts/h) "
|
|---|
| 81 | echo " # QUERY: "$query
|
|---|
| 82 | getdbsetup
|
|---|
| 83 | printprocesslog "DEBUG sendquery QUERY: "$query
|
|---|
| 84 | echo "DEBUG sendquery QUERY: "$query
|
|---|
| 85 | mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
|
|---|
| 86 | if [ "$table" == "AnalysisResultsRunLP" ]
|
|---|
| 87 | then
|
|---|
| 88 | query2=`echo $query | sed -e 's/AnalysisResultsRunLP/AnalysisResultsRunISDC/g'`
|
|---|
| 89 | echo "# QUERY: "$query2
|
|---|
| 90 | mysql -s -u $us --password=$pw --host=$ho factdata_cp20150811 -e " $query2 "
|
|---|
| 91 | fi
|
|---|
| 92 | }
|
|---|
| 93 |
|
|---|
| 94 | ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
|
|---|
| 95 | ontime2=" fOnTimeAfterCuts "
|
|---|
| 96 | ontimeif=" IF(ISNULL(fEffectiveOn), fOnTimeAfterCuts, TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn) "
|
|---|
| 97 | from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
|
|---|
| 98 | # time range and source
|
|---|
| 99 | where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$start" AND "$stop
|
|---|
| 100 | # some sanity checks
|
|---|
| 101 | where=$where" AND fRunTypeKey=1 AND NOT ISNULL(fNumSigEvts) AND NOT ISNULL(fNumBgEvts) AND NOT fRunStart='0000-00-00 00:00:00' AND NOT fRunStop='0000-00-00 00:00:00' "
|
|---|
| 102 | where=$where" AND fZenithDistanceMax<"$zdcut" AND fThresholdMedian<"$thcut" AND fMoonZenithDistance>"$dark
|
|---|
| 103 | where=$where" AND (fTNGDust < "$dust" OR isnull(fTNGDust)) "
|
|---|
| 104 |
|
|---|
| 105 | # DataCheck (old)
|
|---|
| 106 | ##data with old feedback and/or different bias voltage
|
|---|
| 107 | #query=$query" AND fNight>20120420 AND NOT fNight IN (20120406,20120410,20120503) AND"
|
|---|
| 108 | ## broken bias channel
|
|---|
| 109 | #query=$query" NOT fNight BETWEEN 20121206 AND 20130110"
|
|---|
| 110 | # bg-rate cut
|
|---|
| 111 | zdparam=" pow(0.753833*cos(Radians(fZenithDistanceMean)), 7.647435)*exp(-5.753686*pow(Radians(fZenithDistanceMean),2.089609))"
|
|---|
| 112 | thparam=" pow((if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)-329.4203),2)*(-0.0000002044803) "
|
|---|
| 113 | param=" (fNumEvtsAfterBgCuts/5-fNumSigEvts)/fOnTimeAfterCuts - "$zdparam" - "$thparam" "
|
|---|
| 114 | dchold=" -0.085 < ("$param") "
|
|---|
| 115 | dchold=$dchold" AND 0.25 > ("$param") "
|
|---|
| 116 | # Datacheck (new) -> combine
|
|---|
| 117 | dchval=" fNumEvtsAfterBgCuts/(1.41*POW(fZenithDistanceMean*PI()/180,2)+0.975)/(-7.53e-12*POW(10, LOG10(fThresholdMinSet)*3.69)+1.035)/TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))/fEffectiveOn "
|
|---|
| 118 | # some semi-automatic datacheck
|
|---|
| 119 | datacheck=" AND (("$dchval" BETWEEN 0.8 AND 1.7 AND fNight BETWEEN 20140520 AND 20150131) " #A
|
|---|
| 120 | datacheck=$datacheck" OR ("$dchval" BETWEEN 0.4 AND 1.6 AND fNight BETWEEN 20150201 AND 20150715) " #B
|
|---|
| 121 | datacheck=$datacheck" OR ("$dchval" BETWEEN 0.7 AND 1.4 AND fNight BETWEEN 20150716 AND 20160218) " #C
|
|---|
| 122 | datacheck=$datacheck" OR ("$dchval" BETWEEN 0.5 AND 1.0 AND fNight > 20160220) " #D
|
|---|
| 123 | datacheck=$datacheck" OR ("$dchold" AND fNight<20140520)) " #old
|
|---|
| 124 |
|
|---|
| 125 |
|
|---|
| 126 | if [ $dch -eq 0 ]
|
|---|
| 127 | then
|
|---|
| 128 | datacheck=""
|
|---|
| 129 | fi
|
|---|
| 130 |
|
|---|
| 131 | case $time in
|
|---|
| 132 | mjd) timeselect="Mjd(Min(fRunStart)) AS start, Mjd(MAX(fRunStop)) AS stop, "
|
|---|
| 133 | timeselect2=" Mjd(MIN(o.start)) AS start, Mjd(MAX(o.stop)) AS stop, "
|
|---|
| 134 | break;;
|
|---|
| 135 | unix) timeselect="Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')) AS start, Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) AS stop, "
|
|---|
| 136 | timeselect2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start, Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop, "
|
|---|
| 137 | break;;
|
|---|
| 138 | *) timeselect="MIN(fRunStart) AS start, MAX(fRunStop) AS stop, "
|
|---|
| 139 | timeselect2=" MIN(o.start) AS start, MAX(o.stop) as stop, "
|
|---|
| 140 | break;;
|
|---|
| 141 | esac
|
|---|
| 142 |
|
|---|
| 143 | # todo:
|
|---|
| 144 | # implement login
|
|---|
| 145 | # implement for internal use
|
|---|
| 146 | # table
|
|---|
| 147 | # more binnings (insert box for minutes? period, 10d?, weekly?)
|
|---|
| 148 | # zd limit
|
|---|
| 149 | # threshold limit
|
|---|
| 150 | # datacheck
|
|---|
| 151 |
|
|---|
| 152 | if [ $bin -eq -1 ]
|
|---|
| 153 | then
|
|---|
| 154 | #query="SELECT Min(fRunStart), Max(fRunStop), Sum(fNumExcEvts)/Sum(fOnTimeAfterCuts)*3600, if ((Sum(fNumSigEvts) + 0.2*0.2*Sum(fNumBgEvts))<0, 0, Sqrt(Sum(fNumSigEvts) + 0.2*0.2*Sum(fNumBgEvts))) FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) WHERE fSourceKey="$source" AND fNight BETWEEN "$start" AND "$stop" GROUP BY fNight HAVING SUM(fOnTimeAfterCuts)*60>20"
|
|---|
| 155 | query="SELECT "$timeselect" SUM(fNumExcEvts)/SUM("$ontimeif")*3600 AS excrate, "
|
|---|
| 156 | query=$query" ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontimeif")*3600 AS excerr, "
|
|---|
| 157 | query=$query" IF (SUM(fNumSigEvts)>10 AND SUM(fNumBgEvts)>10, LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)), IF (SUM(fNumSigEvts)-SUM(fNumBgEvts)>10, concat('(', LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)), ')'), '-')) AS significance, "
|
|---|
| 158 | query=$query" LiMa(SUM(fNumSigEvts), SUM(fNumBgEvts)) AS significance2, "
|
|---|
| 159 | query=$query" Avg(fZenithDistanceMean) as zd, "
|
|---|
| 160 | query=$query" Avg(fThresholdMedian) as th, "
|
|---|
| 161 | query=$query" SUM(fNumSigEvts)/SUM("$ontimeif")*3600 AS sigrate, "
|
|---|
| 162 | query=$query" SUM(fNumBgEvts)/SUM("$ontimeif")*3600 AS bgrate, "
|
|---|
| 163 | #query=$query" ((1.41*POW(fZenithDistanceMean*PI()/180,2)+0.975)/(-7.53e-12*POW(10, LOG10(fThresholdMinSet)*3.69)+1.035) ) AS factor "
|
|---|
| 164 | query=$query" (SUM(fNumExcEvts)/SUM(fNumExcEvts/((1.41*POW(fZenithDistanceMean*PI()/180,2)+0.975)/(-7.53e-12*POW(10, LOG10(fThresholdMinSet)*3.69)+1.035) ))) AS factor, "
|
|---|
| 165 | query=$query" SUM(fNumEvtsAfterBgCuts/(1.41*POW(fZenithDistanceMean*PI()/180,2)+0.975)/(-7.53e-12*POW(10, LOG10(fThresholdMinSet)*3.69)+1.035) )/SUM(TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn) AS bgfactor "
|
|---|
| 166 | query=$query$from$where$datacheck" GROUP BY fNight HAVING SUM("$ontimeif")/60>20 ORDER BY fNight "
|
|---|
| 167 | else
|
|---|
| 168 | # query to get excess rate and significance from DB
|
|---|
| 169 | query="SELECT "$timeselect2" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 2) AS excrate, "
|
|---|
| 170 | query=$query" ROUND(ExcErr(SUM(o.sigevts),SUM(o.bgevts))/SUM(o.ot)*3600,2) AS err "
|
|---|
| 171 | #query=$query" ROUND(SUM(o.ot)/60.,1) AS 'ontime [min]' "
|
|---|
| 172 | #query=$query" IF (SUM(o.sigevts)>10 AND SUM(o.bgevts)>10, LiMa(SUM(o.sigevts), SUM(o.bgevts)), IF (SUM(o.sigevts)-SUM(o.bgevts)>10, CONCAT('(', LiMa(SUM(o.sigevts), SUM(o.bgevts)), ')'), '-')) AS significance "
|
|---|
| 173 | query=$query" FROM (SELECT fNight, @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "
|
|---|
| 174 | query=$query" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
|
|---|
| 175 | query=$query" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
|
|---|
| 176 | query=$query$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
|
|---|
| 177 | query=$query$where$datacheck" ORDER BY fRunStart) o GROUP BY block ORDER BY start"
|
|---|
| 178 | fi
|
|---|
| 179 | #echo $query
|
|---|
| 180 |
|
|---|
| 181 | printinfo | mail -s 'test internal downloading data ' -c dorner@astro.uni-wuerzburg.de -r dorner@astro.uni-wuerzburg.de $email
|
|---|
| 182 | #printinfo
|
|---|
| 183 |
|
|---|
| 184 | finish
|
|---|
| 185 |
|
|---|
