| 1 | #!/bin/bash
|
|---|
| 2 |
|
|---|
| 3 | # ---------------------------------------------------------------- #
|
|---|
| 4 | # README README README README README README README README README #
|
|---|
| 5 | # ---------------------------------------------------------------- #
|
|---|
| 6 | # #
|
|---|
| 7 | # To use this script, you need #
|
|---|
| 8 | # - a computer with access to the FACT database in La Palma #
|
|---|
| 9 | # - a file with the password of a valid mysql-user #
|
|---|
| 10 | # - to define the setup below for #
|
|---|
| 11 | # a) the DB access #
|
|---|
| 12 | # b) the data you want to have #
|
|---|
| 13 | # #
|
|---|
| 14 | # To define the setup, search for SETUP in this script and #
|
|---|
| 15 | # read the details there #
|
|---|
| 16 | # #
|
|---|
| 17 | # Per data request, you get up to 3 files: #
|
|---|
| 18 | # *_internal.dat #
|
|---|
| 19 | # *_collaborators.dat #
|
|---|
| 20 | # *_external.dat (only if binning is 20min or nightly) #
|
|---|
| 21 | # #
|
|---|
| 22 | # Please have in mind that this started as a tool for myself, then #
|
|---|
| 23 | # others started using it. Also the script is not yet finalized. #
|
|---|
| 24 | # In case you find problems and/or have a feature request, please #
|
|---|
| 25 | # send and email to dorner@astro.uni-wuerzburg.de #
|
|---|
| 26 | # #
|
|---|
| 27 | # ---------------------------------------------------------------- #
|
|---|
| 28 | # README README README README README README README README README #
|
|---|
| 29 | # ---------------------------------------------------------------- #
|
|---|
| 30 |
|
|---|
| 31 |
|
|---|
| 32 |
|
|---|
| 33 |
|
|---|
| 34 |
|
|---|
| 35 | # ToDo (notes DD):
|
|---|
| 36 | # ----------------
|
|---|
| 37 | # must-have
|
|---|
| 38 | # - fix bug for timeunit=timestamp (min-binning)
|
|---|
| 39 | # - add email sending and return values
|
|---|
| 40 | # - update function for zd/th-correction
|
|---|
| 41 | # - update CU for QLA
|
|---|
| 42 | # - add CU for ISDC analysis
|
|---|
| 43 | # - add < 20121212 data for QLA
|
|---|
| 44 | # - check crab flux
|
|---|
| 45 | # nice-to-have
|
|---|
| 46 | # - add E2dNdE
|
|---|
| 47 | # - functionality to determine start time for seaon-binning
|
|---|
| 48 | # - offer several predefined datachecks?
|
|---|
| 49 |
|
|---|
| 50 | #
|
|---|
| 51 | # content of files (wish list):
|
|---|
| 52 | # -----------------------------
|
|---|
| 53 | # REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
|
|---|
| 54 | #
|
|---|
| 55 | # internal
|
|---|
| 56 | # --------
|
|---|
| 57 | # time: time, delta time, start, stop, ontime
|
|---|
| 58 | # flux: excrate, excerr, corrate, corerr, CU CUerr, flux, fluxerr,
|
|---|
| 59 | # other info on flux: signif, cu-factor, num exc, num sig, num bg
|
|---|
| 60 | # other info: zd th R750cor R750ref
|
|---|
| 61 | #
|
|---|
| 62 | # external (allow only 20min and nightly binning)
|
|---|
| 63 | # --------
|
|---|
| 64 | # time: time, delta time, start, stop
|
|---|
| 65 | # flux: excrate, excerr
|
|---|
| 66 | #
|
|---|
| 67 | # collaborators
|
|---|
| 68 | # -------------
|
|---|
| 69 | # time: time, delta time, start, stop, ontime
|
|---|
| 70 | # flux: excrate, excerr, corrate, corerr, flux, flux-err, significance
|
|---|
| 71 | #
|
|---|
| 72 | # additional information to put:
|
|---|
| 73 | # ------------------------------
|
|---|
| 74 | # timestamp of creation
|
|---|
| 75 | # query (for debugging / answering questions)
|
|---|
| 76 | # policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
|
|---|
| 77 | #
|
|---|
| 78 |
|
|---|
| 79 |
|
|---|
| 80 | function print_policy()
|
|---|
| 81 | {
|
|---|
| 82 | echo "# Data Usage Policy: "
|
|---|
| 83 | if [ "$expert" == "no" ]
|
|---|
| 84 | then
|
|---|
| 85 | echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
|
|---|
| 86 | echo "# paper and the quick look analysis website. "
|
|---|
| 87 | echo "# References: "
|
|---|
| 88 | echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
|
|---|
| 89 | echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
|
|---|
| 90 | echo "# FACT Performance Paper: href='http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
|
|---|
| 91 | echo "# href='http://iopscience.iop.org/1748-0221/9/10/P10012"
|
|---|
| 92 | echo "# FACT quick look analysis: https://fact-project.org/monitoring"
|
|---|
| 93 | echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
|
|---|
| 94 | echo "# If you intend to use data or information from this website, please let us know for reference."
|
|---|
| 95 | else
|
|---|
| 96 | echo "# As a member or associated member of the FACT Collaboration, you have access to internal information."
|
|---|
| 97 | echo "# Any publication using FACT internal information has to have the full FACT author list."
|
|---|
| 98 | fi
|
|---|
| 99 | }
|
|---|
| 100 |
|
|---|
| 101 |
|
|---|
| 102 |
|
|---|
| 103 | function get_results()
|
|---|
| 104 | {
|
|---|
| 105 | # some query parts
|
|---|
| 106 |
|
|---|
| 107 | # some numbers for flux calculation
|
|---|
| 108 | crabflux="3.37e-11"
|
|---|
| 109 | fluxprec=13
|
|---|
| 110 | crabflux="3.37"
|
|---|
| 111 | fluxprec=2
|
|---|
| 112 |
|
|---|
| 113 | # some names and definitions needed several times below
|
|---|
| 114 | # ontime
|
|---|
| 115 | ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
|
|---|
| 116 | ontime2=" fOnTimeAfterCuts "
|
|---|
| 117 | ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
|
|---|
| 118 | # zd and threshold
|
|---|
| 119 | zenith="fZenithDistance"
|
|---|
| 120 | thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
|
|---|
| 121 | # correction factor for excess rate (formula by TB)
|
|---|
| 122 | correvts=" fNumExcEvts*(pow(cos("$zenith"Mean*PI()/180),3)+14.8/21.9*pow(sin(2*"$zenith"Mean*PI()/180),5))/((1-0.00124/1.21*("$thresh"-500)*("$thresh">=500))) "
|
|---|
| 123 | # conversion to CU (determined by DD for QLA)
|
|---|
| 124 | # https://www.fact-project.org/logbook/showthread.php?tid=4927
|
|---|
| 125 | cufactor="CUQLA(fNight)" # missing: ISDC analysis
|
|---|
| 126 | # some calculations
|
|---|
| 127 | excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
|
|---|
| 128 | CU="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
|
|---|
| 129 | CUerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
|
|---|
| 130 | excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
|
|---|
| 131 | CU2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
|
|---|
| 132 | CUerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
|
|---|
| 133 |
|
|---|
| 134 | # columns to be selected
|
|---|
| 135 | # for night-binning
|
|---|
| 136 | ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
|
|---|
| 137 | excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
|
|---|
| 138 | significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
|
|---|
| 139 | numexc="Sum(fNumExcEvts) AS numexc"
|
|---|
| 140 | numsig="Sum(fNumSigEvts) AS numsig"
|
|---|
| 141 | numbg="Sum(fNumBgEvts) AS numbg"
|
|---|
| 142 | excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
|
|---|
| 143 | correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
|
|---|
| 144 | correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
|
|---|
| 145 | cu=" ROUND("$CU", 2) AS cu"
|
|---|
| 146 | cuerr=" ROUND("$CUerr", 2) AS cuerr"
|
|---|
| 147 | flux="ROUND("$CU" * "$crabflux", 2) AS flux"
|
|---|
| 148 | fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
|
|---|
| 149 | # for minute binning
|
|---|
| 150 | ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
|
|---|
| 151 | excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
|
|---|
| 152 | significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
|
|---|
| 153 | numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
|
|---|
| 154 | numsig2="Sum(o.sigevts) AS numsig"
|
|---|
| 155 | numbg2="Sum(o.bgevts) AS numbg"
|
|---|
| 156 | excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
|
|---|
| 157 | correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
|
|---|
| 158 | correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
|
|---|
| 159 | cu2=" ROUND("$CU2", 2) AS cu"
|
|---|
| 160 | cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
|
|---|
| 161 | flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
|
|---|
| 162 | fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
|
|---|
| 163 |
|
|---|
| 164 | case $timeunit in
|
|---|
| 165 | mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
|
|---|
| 166 | start=" Mjd(MIN(fRunStart)) AS start"
|
|---|
| 167 | stop=" Mjd(MAX(fRunStop)) AS stop"
|
|---|
| 168 | deltat=$delta" AS deltat"
|
|---|
| 169 | time=" Mjd(MIN(fRunStart))+"$delta" AS time"
|
|---|
| 170 | delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
|
|---|
| 171 | start2=" Mjd(MIN(o.start)) AS start"
|
|---|
| 172 | stop2=" Mjd(MAX(o.stop)) AS stop"
|
|---|
| 173 | deltat2=$delta2" AS deltat"
|
|---|
| 174 | time2=" Mjd(MIN(o.start))+"$delta2" AS time"
|
|---|
| 175 | ;;
|
|---|
| 176 | unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
|
|---|
| 177 | start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
|
|---|
| 178 | stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
|
|---|
| 179 | deltat=$delta" AS deltat"
|
|---|
| 180 | time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
|
|---|
| 181 | delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
|
|---|
| 182 | startstop2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
|
|---|
| 183 | startstop2=$starstop2" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
|
|---|
| 184 | deltat2=$delta2" AS deltat"
|
|---|
| 185 | time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
|
|---|
| 186 | ;;
|
|---|
| 187 | *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
|
|---|
| 188 | start=" MIN(fRunStart) AS start"
|
|---|
| 189 | stop=" MAX(fRunStop) AS stop"
|
|---|
| 190 | deltat=$delta" AS deltat"
|
|---|
| 191 | time=" addtime(MIN(fRunStart), "$delta") AS time"
|
|---|
| 192 | delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
|
|---|
| 193 | start2=" MIN(o.start) AS start"
|
|---|
| 194 | stop2=" MAX(o.stop) AS stop"
|
|---|
| 195 | deltat2=$delta" AS deltat"
|
|---|
| 196 | time2=" addtime(MIN(o.start), "$delta2") AS time"
|
|---|
| 197 | ;;
|
|---|
| 198 | esac
|
|---|
| 199 |
|
|---|
| 200 | # from and join of query
|
|---|
| 201 | from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
|
|---|
| 202 |
|
|---|
| 203 | # data check based on artificial trigger rate
|
|---|
| 204 | # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
|
|---|
| 205 | #dch=" AND fR750Cor/fR750Ref >0.93 "
|
|---|
| 206 | dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
|
|---|
| 207 |
|
|---|
| 208 | # put together where-clause of query
|
|---|
| 209 | # time range and source
|
|---|
| 210 | where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
|
|---|
| 211 | where=$where" AND NOT ISNULL(fNumExcEvts) "
|
|---|
| 212 | # some sanity checks
|
|---|
| 213 | where=$where" AND fRunTypeKey=1 "
|
|---|
| 214 | # zd cut
|
|---|
| 215 | if [ "$zdmax" != "" ]
|
|---|
| 216 | then
|
|---|
| 217 | where=$where" AND fZenithDistanceMax < "$zdmax
|
|---|
| 218 | fi
|
|---|
| 219 | # th cut
|
|---|
| 220 | if [ "$thmax" != "" ]
|
|---|
| 221 | then
|
|---|
| 222 | where=$where" AND "$thresh" < "$thmax
|
|---|
| 223 | fi
|
|---|
| 224 | # dust cut
|
|---|
| 225 | if [ "$dust" != "" ]
|
|---|
| 226 | then
|
|---|
| 227 | where=$where" AND fTNGDust<"$dust
|
|---|
| 228 | fi
|
|---|
| 229 | # light condition cut
|
|---|
| 230 | if [ "$light" == "no moon" ]
|
|---|
| 231 | then
|
|---|
| 232 | where=$where" AND fZenithDistanceMoon>90"
|
|---|
| 233 | fi
|
|---|
| 234 | if [ "$light" == "dark" ]
|
|---|
| 235 | then
|
|---|
| 236 | where=$where" AND fMoonZenithDistance>90 AND fSunZenithDistance>108 "
|
|---|
| 237 | fi
|
|---|
| 238 | querybase=$from$where
|
|---|
| 239 |
|
|---|
| 240 | if [ "$usedch" == "yes" ]
|
|---|
| 241 | then
|
|---|
| 242 | if [ "$dch" == "" ]
|
|---|
| 243 | then
|
|---|
| 244 | querydch=$dchstd
|
|---|
| 245 | else
|
|---|
| 246 | echo "you are using for datacheck: "$dch
|
|---|
| 247 | querydch=$dch
|
|---|
| 248 | fi
|
|---|
| 249 | fi
|
|---|
| 250 |
|
|---|
| 251 |
|
|---|
| 252 | if [ $bin -le 0 ]
|
|---|
| 253 | then
|
|---|
| 254 | num="#bin"
|
|---|
| 255 | # first part of the query
|
|---|
| 256 | querystart="SELECT "
|
|---|
| 257 | if [ $bin -eq 0 ]
|
|---|
| 258 | then
|
|---|
| 259 | querystart=$querystart" fPeriod AS num, "
|
|---|
| 260 | else
|
|---|
| 261 | querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
|
|---|
| 262 | fi
|
|---|
| 263 | querystart=$querystart" "$time", "$start", "$stop", "
|
|---|
| 264 |
|
|---|
| 265 | # final part of the query
|
|---|
| 266 | queryend=" GROUP BY num "
|
|---|
| 267 | if [ "$ontimelimit" = "" ]
|
|---|
| 268 | then
|
|---|
| 269 | queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY num " # 20 min
|
|---|
| 270 | else
|
|---|
| 271 | queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY num "
|
|---|
| 272 | fi
|
|---|
| 273 |
|
|---|
| 274 | # internal
|
|---|
| 275 | queryint=$querystart
|
|---|
| 276 | queryint=$queryint" "$excrate", "$correxcrate", "$cu", "$flux", "
|
|---|
| 277 | queryint=$queryint" "$deltat", "$ontime", "
|
|---|
| 278 | queryint=$queryint" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
|
|---|
| 279 | queryint=$queryint" "$significance", "
|
|---|
| 280 | queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
|
|---|
| 281 | queryint=$queryint" "$numexc", "$numsig", "$numbg", "
|
|---|
| 282 | queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
|
|---|
| 283 | queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
|
|---|
| 284 | queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
|
|---|
| 285 | queryint=$queryint" "$querybase" "$querydch" "$queryend
|
|---|
| 286 |
|
|---|
| 287 | # for collaborators
|
|---|
| 288 | querycol=$querystart
|
|---|
| 289 | querycol=$querycol" "$excrate", "$correxcrate", "$cu", "$flux", "
|
|---|
| 290 | querycol=$querycol" "$deltat", "$ontime", "
|
|---|
| 291 | querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
|
|---|
| 292 | querycol=$querycol" "$significance
|
|---|
| 293 | querycol=$querycol" "$querybase" "$querydch" "$queryend
|
|---|
| 294 |
|
|---|
| 295 | # external
|
|---|
| 296 | # no datacheck applied for external files
|
|---|
| 297 | queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
|
|---|
| 298 |
|
|---|
| 299 | else
|
|---|
| 300 | num=
|
|---|
| 301 | # first part of the query
|
|---|
| 302 | querystart="SELECT "
|
|---|
| 303 | querystart=$querystart" "$time2", "$start2", "$stop2", "
|
|---|
| 304 |
|
|---|
| 305 | # final part of the query
|
|---|
| 306 | querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
|
|---|
| 307 | querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
|
|---|
| 308 | querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
|
|---|
| 309 | querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
|
|---|
| 310 | querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
|
|---|
| 311 | querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
|
|---|
| 312 | querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
|
|---|
| 313 | querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
|
|---|
| 314 |
|
|---|
| 315 | # internal
|
|---|
| 316 | queryint=$querystart
|
|---|
| 317 | queryint=$queryint" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
|
|---|
| 318 | queryint=$queryint" "$deltat2", "$ontime2", "
|
|---|
| 319 | queryint=$queryint" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
|
|---|
| 320 | queryint=$queryint" "$significance2", "
|
|---|
| 321 | queryint=$queryint" avg(o.night) AS night, "
|
|---|
| 322 | queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
|
|---|
| 323 | queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
|
|---|
| 324 | queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
|
|---|
| 325 | queryint=$queryint" "$querybase
|
|---|
| 326 |
|
|---|
| 327 | # for collaborators
|
|---|
| 328 | querycol=$querystart
|
|---|
| 329 | querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
|
|---|
| 330 | querycol=$querycol" "$deltat2", "$ontime2", "
|
|---|
| 331 | querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
|
|---|
| 332 | querycol=$querycol" "$significance2
|
|---|
| 333 | querycol=$querycol" "$querybase
|
|---|
| 334 |
|
|---|
| 335 | # external
|
|---|
| 336 | queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
|
|---|
| 337 |
|
|---|
| 338 | fi
|
|---|
| 339 |
|
|---|
| 340 | # write file for externals only for allowed binnings
|
|---|
| 341 | if [ $bin -eq 20 ] || [ $bin -eq -1 ]
|
|---|
| 342 | then
|
|---|
| 343 | fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
|
|---|
| 344 | if [ "$overwrite" = "yes" ]
|
|---|
| 345 | then
|
|---|
| 346 | echo "creating "$fileext" ..."
|
|---|
| 347 | echo "# This file was created at "`date` > $fileext
|
|---|
| 348 | print_policy >> $fileext
|
|---|
| 349 | fi
|
|---|
| 350 | headerext="# "$num" time["$timeunit"] start["$timeunit"] stop["$timeunit"] excrate[evts/h] delta_time["$timeunit"] excrate_err[evts/h] "
|
|---|
| 351 | echo $headerext >> $fileext
|
|---|
| 352 | #echo "$queryext"
|
|---|
| 353 | mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryext" >> $fileext
|
|---|
| 354 | #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryext"
|
|---|
| 355 | fi
|
|---|
| 356 | if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
|
|---|
| 357 | then
|
|---|
| 358 | return
|
|---|
| 359 | fi
|
|---|
| 360 |
|
|---|
| 361 | fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
|
|---|
| 362 | if [ "$overwrite" = "yes" ]
|
|---|
| 363 | then
|
|---|
| 364 | echo "creating "$fileint" ..."
|
|---|
| 365 | echo "# This file was created at "`date` > $fileint
|
|---|
| 366 | echo "#" >> $fileint
|
|---|
| 367 | print_policy >> $fileint
|
|---|
| 368 | echo "#" >> $fileint
|
|---|
| 369 | echo "# The following query was used: " >> $fileint
|
|---|
| 370 | echo "# "$queryint >> $fileint
|
|---|
| 371 | echo "#" >> $fileint
|
|---|
| 372 | fi
|
|---|
| 373 | headerint="# "$num" time["$timeunit"] start["$timeunit"] stop["$timeunit"] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time["$timeunit"] ontime[min]"
|
|---|
| 374 | headerint=$headerint" excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance night num_exc num_sig num_bg "
|
|---|
| 375 | headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
|
|---|
| 376 | echo $headerint >> $fileint
|
|---|
| 377 | #echo "$queryint"
|
|---|
| 378 | mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryint" >> $fileint
|
|---|
| 379 | #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryint"
|
|---|
| 380 | if [ "$mode"="auto" ]
|
|---|
| 381 | then
|
|---|
| 382 | return
|
|---|
| 383 | fi
|
|---|
| 384 |
|
|---|
| 385 | filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
|
|---|
| 386 | if [ "$overwrite" = "yes" ]
|
|---|
| 387 | then
|
|---|
| 388 | echo "creating "$filecol" ..."
|
|---|
| 389 | echo "# This file was created at "`date` > $filecol
|
|---|
| 390 | echo "#" >> $filecol
|
|---|
| 391 | print_policy >> $filecol
|
|---|
| 392 | echo "#" >> $filecol
|
|---|
| 393 | echo "# The following query was used: " >> $filecol
|
|---|
| 394 | echo "# "$querycol >> $filecol
|
|---|
| 395 | echo "#" >> $filecol
|
|---|
| 396 | fi
|
|---|
| 397 | headercol="# "$num" time["$timeunit"] start["$timeunit"] stop["$timeunit"] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time["$timeunit"] ontime[min]"
|
|---|
| 398 | headercol=$headercol" excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance "
|
|---|
| 399 | echo $headercol >> $filecol
|
|---|
| 400 | #echo "$querycol"
|
|---|
| 401 | mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$querycol" >> $filecol
|
|---|
| 402 | #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$querycol
|
|---|
| 403 |
|
|---|
| 404 | }
|
|---|
| 405 |
|
|---|
| 406 | # evaluation of command line options (for usage with download.php)
|
|---|
| 407 |
|
|---|
| 408 | if [ ${#@} -eq 13 ]
|
|---|
| 409 | then
|
|---|
| 410 | #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
|
|---|
| 411 | mode="auto"
|
|---|
| 412 | overwrite="yes"
|
|---|
| 413 | # setup
|
|---|
| 414 | path=`dirname $0`
|
|---|
| 415 | datapath=$path"/data"
|
|---|
| 416 | sqlpw=/home/$USER/.mysql.pw2
|
|---|
| 417 | host=10.0.100.21
|
|---|
| 418 | dbname=factdata
|
|---|
| 419 | nightmin=$1
|
|---|
| 420 | nightmax=$2
|
|---|
| 421 | source=$3
|
|---|
| 422 | bin=$4
|
|---|
| 423 | email=$5
|
|---|
| 424 | table=$6
|
|---|
| 425 | timeunit=$7
|
|---|
| 426 | expert=$8
|
|---|
| 427 | usedch=$9 # novalue gives same result as no
|
|---|
| 428 | if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
|
|---|
| 429 | then
|
|---|
| 430 | zdmax=${10}
|
|---|
| 431 | fi
|
|---|
| 432 | if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
|
|---|
| 433 | then
|
|---|
| 434 | thmax=${11}
|
|---|
| 435 | fi
|
|---|
| 436 | if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
|
|---|
| 437 | then
|
|---|
| 438 | light=${12}
|
|---|
| 439 | fi
|
|---|
| 440 | if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
|
|---|
| 441 | then
|
|---|
| 442 | dust=${13}
|
|---|
| 443 | fi
|
|---|
| 444 | name=`echo $email | sed -e 's/@/-at-/'`
|
|---|
| 445 | get_results
|
|---|
| 446 |
|
|---|
| 447 | if [ "$expert" == "yes" ]
|
|---|
| 448 | then
|
|---|
| 449 | echo $fileint
|
|---|
| 450 | else
|
|---|
| 451 | echo $fileext
|
|---|
| 452 | fi
|
|---|
| 453 |
|
|---|
| 454 | # next: light, dust
|
|---|
| 455 | # implement all options
|
|---|
| 456 | # make dch swi
|
|---|
| 457 |
|
|---|
| 458 | exit
|
|---|
| 459 | fi
|
|---|
| 460 |
|
|---|
| 461 |
|
|---|
| 462 |
|
|---|
| 463 | # -------------------------------------------------------------------------------------- #
|
|---|
| 464 | # SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
|
|---|
| 465 | # -------------------------------------------------------------------------------------- #
|
|---|
| 466 | # #
|
|---|
| 467 | # The lines below define the basic setup for the database and give examples and #
|
|---|
| 468 | # explanations for the various options available. #
|
|---|
| 469 | # The request of the data itself is done with a smaller setup further down. #
|
|---|
| 470 | # #
|
|---|
| 471 | # -------------------------------------------------------------------------------------- #
|
|---|
| 472 | #
|
|---|
| 473 | # ----------
|
|---|
| 474 | # DB SETUP
|
|---|
| 475 | # ----------
|
|---|
| 476 | # path to file with mysql password
|
|---|
| 477 | sqlpw=/home/$USER/.mysql.pw
|
|---|
| 478 | # host of mysql server with FACT DB
|
|---|
| 479 | #host=lp-fact # ISDC
|
|---|
| 480 | host=10.0.100.21 # LP or LP via vpn
|
|---|
| 481 | #host=localhost # your local machine in case you have a copy of DB
|
|---|
| 482 | # name of database
|
|---|
| 483 | dbname=factdata
|
|---|
| 484 | #
|
|---|
| 485 | # -------------
|
|---|
| 486 | # BASIC SETUP
|
|---|
| 487 | # -------------
|
|---|
| 488 | # output path
|
|---|
| 489 | path=`dirname $0`
|
|---|
| 490 | datapath=$path"/data"
|
|---|
| 491 | # create directory for data files
|
|---|
| 492 | if ! [ -e $datapath ]
|
|---|
| 493 | then
|
|---|
| 494 | mkdir $datapath
|
|---|
| 495 | fi
|
|---|
| 496 | # time unit
|
|---|
| 497 | #timeunit=timestamp # default
|
|---|
| 498 | #timeunit=unix
|
|---|
| 499 | timeunit=mjd
|
|---|
| 500 | # time binning
|
|---|
| 501 | # positive values: minutes
|
|---|
| 502 | # negative values: days
|
|---|
| 503 | # special case 0: period
|
|---|
| 504 | # for season binning choose -365 and according start date
|
|---|
| 505 | #bin=20 # minutes
|
|---|
| 506 | #bin=0 # period
|
|---|
| 507 | bin=-1 # nightly
|
|---|
| 508 | #bin=-365 # yearly
|
|---|
| 509 | # choose analysis
|
|---|
| 510 | #table="AnalysisResultsAllQLA" # N/A
|
|---|
| 511 | table="AnalysisResultsRunLP" # QLA
|
|---|
| 512 | #table="AnalysisResultsRunISDC" # ISDC
|
|---|
| 513 | # time range
|
|---|
| 514 | nightmin=20111115
|
|---|
| 515 | nightmax=20201231
|
|---|
| 516 | # overwrite dataset file?
|
|---|
| 517 | # (useful to combine different binnings in one file -> set to "no")
|
|---|
| 518 | overwrite="yes"
|
|---|
| 519 | # optional: require minimal ontime per bin (default 20 min)
|
|---|
| 520 | #ontimelimit=30 # 30 min
|
|---|
| 521 | ontimelimit= # default 20 min
|
|---|
| 522 | # data quality selection
|
|---|
| 523 | # if you explicitely don't want a datacheck, you can comment the following line
|
|---|
| 524 | usedch="yes"
|
|---|
| 525 | # use your own datacheck instead
|
|---|
| 526 | # use a line like the following defining your own data quality selection cut
|
|---|
| 527 | #dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
|
|---|
| 528 | # apply additional predefined cuts
|
|---|
| 529 | # light conditions
|
|---|
| 530 | #light="no moon" # only data with no moon (but twilight allowed)
|
|---|
| 531 | #light="dark" # only dark night data
|
|---|
| 532 | # TNG dust - cut away data with calima
|
|---|
| 533 | #dust=1
|
|---|
| 534 | #dust=10
|
|---|
| 535 |
|
|---|
| 536 |
|
|---|
| 537 | # -------------------------------------------------------------------------------------- #
|
|---|
| 538 | # SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
|
|---|
| 539 | # -------------------------------------------------------------------------------------- #
|
|---|
| 540 | # #
|
|---|
| 541 | # Adapt the lines below to your needs. #
|
|---|
| 542 | # Overwrite default settings above. #
|
|---|
| 543 | # The data-request is sent with the line 'get_results.' #
|
|---|
| 544 | # Minumum setup: Define source key and name for file. #
|
|---|
| 545 | # The list of source keys can be found at #
|
|---|
| 546 | # https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
|
|---|
| 547 | # More examples can be found further down. #
|
|---|
| 548 | # #
|
|---|
| 549 | # REMARKS: #
|
|---|
| 550 | # - correction of effect of zd and threshold not yet finalized and only valid for QLA #
|
|---|
| 551 | # - no CU-conversion available for ISDC-analysis so far (that for QLA is used instead) #
|
|---|
| 552 | # #
|
|---|
| 553 | # -------------------------------------------------------------------------------------- #
|
|---|
| 554 |
|
|---|
| 555 | # 501 MAGIC
|
|---|
| 556 | source=2
|
|---|
| 557 | name="Mrk501_2014_forMAGIC"
|
|---|
| 558 | bin=-1
|
|---|
| 559 | nightmin=20140714
|
|---|
| 560 | nightmax=20140805
|
|---|
| 561 | get_results
|
|---|
| 562 |
|
|---|
| 563 | bin=30
|
|---|
| 564 | name="Mrk501_2014_forMAGIC30"
|
|---|
| 565 | get_results
|
|---|
| 566 |
|
|---|
| 567 | bin=0
|
|---|
| 568 | name="P"
|
|---|
| 569 | nightmin=20140501
|
|---|
| 570 | nightmax=20140930
|
|---|
| 571 | get_results
|
|---|
| 572 |
|
|---|
| 573 | bin=20
|
|---|
| 574 | nightmin=20140623
|
|---|
| 575 | nightmax=20140623
|
|---|
| 576 | name="Mrk501_test"
|
|---|
| 577 | get_results
|
|---|
| 578 |
|
|---|
| 579 |
|
|---|
| 580 | # end script here
|
|---|
| 581 | exit
|
|---|
| 582 |
|
|---|
| 583 |
|
|---|
| 584 |
|
|---|
| 585 | #
|
|---|
| 586 | # more examples
|
|---|
| 587 | #
|
|---|
| 588 |
|
|---|
| 589 | # Mrk 421
|
|---|
| 590 | source=1
|
|---|
| 591 | name="Mrk421_nightly"
|
|---|
| 592 | bin=-1
|
|---|
| 593 | get_results
|
|---|
| 594 | name="Mrk421_20min"
|
|---|
| 595 | bin=20
|
|---|
| 596 | get_results
|
|---|
| 597 | name="Mrk421_3d"
|
|---|
| 598 | bin=-3
|
|---|
| 599 | get_results
|
|---|
| 600 | name="Mrk421_10d"
|
|---|
| 601 | bin=-10
|
|---|
| 602 | get_results
|
|---|
| 603 | name="Mrk421_period"
|
|---|
| 604 | bin=0
|
|---|
| 605 | get_results
|
|---|
| 606 |
|
|---|
| 607 |
|
|---|
| 608 |
|
|---|
| 609 | # Mrk 501
|
|---|
| 610 | source=2
|
|---|
| 611 | name="Mrk501_nightly"
|
|---|
| 612 | bin=-1
|
|---|
| 613 | get_results
|
|---|
| 614 | name="Mrk501_20min"
|
|---|
| 615 | bin=20
|
|---|
| 616 | get_results
|
|---|
| 617 | name="Mrk501_3d"
|
|---|
| 618 | bin=-3
|
|---|
| 619 | get_results
|
|---|
| 620 | name="Mrk501_10d"
|
|---|
| 621 | bin=-10
|
|---|
| 622 | get_results
|
|---|
| 623 | name="Mrk501_period"
|
|---|
| 624 | bin=0
|
|---|
| 625 | get_results
|
|---|
| 626 |
|
|---|
| 627 |
|
|---|
| 628 |
|
|---|
| 629 | # 2344
|
|---|
| 630 | source=3
|
|---|
| 631 | name="2344_nightly"
|
|---|
| 632 | bin=-1
|
|---|
| 633 | get_results
|
|---|
| 634 | name="2344_20min"
|
|---|
| 635 | bin=20
|
|---|
| 636 | get_results
|
|---|
| 637 | name="2344_period"
|
|---|
| 638 | bin=0
|
|---|
| 639 | get_results
|
|---|
| 640 |
|
|---|
| 641 |
|
|---|
| 642 |
|
|---|
| 643 | # 1959
|
|---|
| 644 | source=7
|
|---|
| 645 | name="1959_nightly"
|
|---|
| 646 | bin=-1
|
|---|
| 647 | get_results
|
|---|
| 648 | name="1959_20min"
|
|---|
| 649 | bin=20
|
|---|
| 650 | get_results
|
|---|
| 651 | name="1959_period"
|
|---|
| 652 | bin=0
|
|---|
| 653 | get_results
|
|---|
| 654 |
|
|---|
| 655 |
|
|---|
| 656 |
|
|---|
| 657 | # 0323
|
|---|
| 658 | source=12
|
|---|
| 659 | name="0323_nightly"
|
|---|
| 660 | bin=-1
|
|---|
| 661 | get_results
|
|---|
| 662 | name="0323_20min"
|
|---|
| 663 | bin=20
|
|---|
| 664 | get_results
|
|---|
| 665 | name="0323_period"
|
|---|
| 666 | bin=0
|
|---|
| 667 | get_results
|
|---|
| 668 |
|
|---|
| 669 |
|
|---|
| 670 |
|
|---|
| 671 | # crab
|
|---|
| 672 | source=5
|
|---|
| 673 | name="Crab_nightly"
|
|---|
| 674 | bin=-1
|
|---|
| 675 | get_results
|
|---|
| 676 | name="Crab_20min"
|
|---|
| 677 | bin=20
|
|---|
| 678 | get_results
|
|---|
| 679 | name="Crab_period"
|
|---|
| 680 | bin=0
|
|---|
| 681 | get_results
|
|---|
| 682 | name="Crab_season"
|
|---|
| 683 | bin=-365
|
|---|
| 684 | nightmin=20110716
|
|---|
| 685 | nightmax=20180716
|
|---|
| 686 | get_results
|
|---|
| 687 |
|
|---|
| 688 |
|
|---|
| 689 |
|
|---|
| 690 | name="1959_2016"
|
|---|
| 691 | source=7
|
|---|
| 692 | bin=-1
|
|---|
| 693 | nightmin=20160201
|
|---|
| 694 | nightmax=20161105
|
|---|
| 695 | get_results
|
|---|
| 696 |
|
|---|
| 697 | name="1959_all_variable"
|
|---|
| 698 | overwrite="no"
|
|---|
| 699 | source=7
|
|---|
| 700 | bin=-365
|
|---|
| 701 | nightmin=20120201
|
|---|
| 702 | nightmax=20130131
|
|---|
| 703 | get_results
|
|---|
| 704 | nightmin=20130201
|
|---|
| 705 | nightmax=20140131
|
|---|
| 706 | get_results
|
|---|
| 707 | nightmin=20140201
|
|---|
| 708 | nightmax=20150131
|
|---|
| 709 | get_results
|
|---|
| 710 | bin=0
|
|---|
| 711 | nightmin=20150201
|
|---|
| 712 | nightmax=20160131
|
|---|
| 713 | get_results
|
|---|
| 714 | bin=-1
|
|---|
| 715 | nightmin=20160201
|
|---|
| 716 | nightmax=20170131
|
|---|
| 717 | get_results
|
|---|
| 718 | bin=0
|
|---|
| 719 | nightmin=20170201
|
|---|
| 720 | nightmax=20180131
|
|---|
| 721 | get_results
|
|---|
| 722 |
|
|---|
| 723 |
|
|---|
| 724 |
|
|---|
| 725 | overwrite="yes"
|
|---|
| 726 | name="1959_all_variable2"
|
|---|
| 727 | overwrite="no"
|
|---|
| 728 | source=7
|
|---|
| 729 | bin=-365
|
|---|
| 730 | nightmin=20120201
|
|---|
| 731 | nightmax=20130131
|
|---|
| 732 | get_results
|
|---|
| 733 | nightmin=20130201
|
|---|
| 734 | nightmax=20140131
|
|---|
| 735 | get_results
|
|---|
| 736 | nightmin=20140201
|
|---|
| 737 | nightmax=20150131
|
|---|
| 738 | get_results
|
|---|
| 739 | bin=0
|
|---|
| 740 | nightmin=20150201
|
|---|
| 741 | nightmax=20160131
|
|---|
| 742 | get_results
|
|---|
| 743 | bin=-1
|
|---|
| 744 | nightmin=20160201
|
|---|
| 745 | nightmax=20160817
|
|---|
| 746 | get_results
|
|---|
| 747 | bin=0
|
|---|
| 748 | nightmin=20160818
|
|---|
| 749 | nightmax=20180131
|
|---|
| 750 | get_results
|
|---|
| 751 |
|
|---|
| 752 |
|
|---|
| 753 |
|
|---|
| 754 | overwrite="yes"
|
|---|
| 755 | bin=0
|
|---|
| 756 | source=3
|
|---|
| 757 | name="2344period"
|
|---|
| 758 | get_results
|
|---|
| 759 |
|
|---|
| 760 |
|
|---|
| 761 |
|
|---|
| 762 | # flare night (HESS)
|
|---|
| 763 | name="Mrk501_10min_flarenight"
|
|---|
| 764 | source=2
|
|---|
| 765 | bin=10
|
|---|
| 766 | nightmin=20140623
|
|---|
| 767 | nightmax=20140623
|
|---|
| 768 | get_results
|
|---|
| 769 |
|
|---|
| 770 |
|
|---|
| 771 |
|
|---|
| 772 | # flare night (HESS)
|
|---|
| 773 | name="Mrk501_5min_flarenight"
|
|---|
| 774 | source=2
|
|---|
| 775 | bin=5
|
|---|
| 776 | nightmin=20140623
|
|---|
| 777 | nightmax=20140623
|
|---|
| 778 | get_results
|
|---|
| 779 |
|
|---|
| 780 |
|
|---|
| 781 |
|
|---|
| 782 |
|
|---|
| 783 | # full sample
|
|---|
| 784 | name="Mrk421_all_nightly"
|
|---|
| 785 | source=1
|
|---|
| 786 | get_results
|
|---|
| 787 |
|
|---|
| 788 | name="Mrk501_all_nightly"
|
|---|
| 789 | source=2
|
|---|
| 790 | get_results
|
|---|
| 791 |
|
|---|
| 792 | name="1959_all_nightly"
|
|---|
| 793 | source=7
|
|---|
| 794 | get_results
|
|---|
| 795 |
|
|---|
| 796 | name="2344_all_nightly"
|
|---|
| 797 | source=3
|
|---|
| 798 | get_results
|
|---|
| 799 |
|
|---|
| 800 |
|
|---|
| 801 |
|
|---|
| 802 | name="HESE20160427"
|
|---|
| 803 | source=19
|
|---|
| 804 | nightmin=20160425
|
|---|
| 805 | bin=-10
|
|---|
| 806 | get_results
|
|---|
| 807 |
|
|---|
| 808 | name="AMON20160731"
|
|---|
| 809 | source=21
|
|---|
| 810 | nightmin=20160730
|
|---|
| 811 | bin=-10
|
|---|
| 812 | get_results
|
|---|
| 813 |
|
|---|
| 814 |
|
|---|
| 815 |
|
|---|
