1 | #!/bin/bash
|
---|
2 |
|
---|
3 | # ---------------------------------------------------------------- #
|
---|
4 | # README README README README README README README README README #
|
---|
5 | # ---------------------------------------------------------------- #
|
---|
6 | # #
|
---|
7 | # To use this script, you need #
|
---|
8 | # - a computer with access to the FACT database in La Palma #
|
---|
9 | # - a file with the password of a valid mysql-user #
|
---|
10 | # - to define the setup below for #
|
---|
11 | # a) the DB access #
|
---|
12 | # b) the data you want to have #
|
---|
13 | # #
|
---|
14 | # To define the setup, search for SETUP in this script and #
|
---|
15 | # read the details there #
|
---|
16 | # #
|
---|
17 | # Per data request, you get up to 3 files: #
|
---|
18 | # *_internal.dat #
|
---|
19 | # *_collaborators.dat #
|
---|
20 | # *_external.dat (only if binning is 20min or nightly) #
|
---|
21 | # #
|
---|
22 | # Please have in mind that this started as a tool for myself, then #
|
---|
23 | # others started using it. Also the script is not yet finalized. #
|
---|
24 | # In case you find problems and/or have a feature request, please #
|
---|
25 | # send and email to dorner@astro.uni-wuerzburg.de #
|
---|
26 | # #
|
---|
27 | # ---------------------------------------------------------------- #
|
---|
28 | # README README README README README README README README README #
|
---|
29 | # ---------------------------------------------------------------- #
|
---|
30 |
|
---|
31 |
|
---|
32 |
|
---|
33 |
|
---|
34 |
|
---|
35 | # ToDo (notes DD):
|
---|
36 | # ----------------
|
---|
37 | # - finalize policy and references
|
---|
38 | # must-have
|
---|
39 | # - update function for zd/th-correction
|
---|
40 | # - update CU for QLA
|
---|
41 | # - add CU for ISDC analysis
|
---|
42 | # - add < 20121212 data for QLA
|
---|
43 | # - check crab flux
|
---|
44 | # nice-to-have
|
---|
45 | # - add E2dNdE
|
---|
46 | # - functionality to determine start time for seaon-binning
|
---|
47 | # - offer several predefined datachecks?
|
---|
48 | # - file as attachment to email ofr download.php
|
---|
49 |
|
---|
50 | #
|
---|
51 | # content of files (wish list):
|
---|
52 | # -----------------------------
|
---|
53 | # REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
|
---|
54 | #
|
---|
55 | # internal
|
---|
56 | # --------
|
---|
57 | # time: time, delta time, start, stop, ontime
|
---|
58 | # flux: excrate, excerr, corrate, corerr, CU CUerr, flux, fluxerr,
|
---|
59 | # other info on flux: signif, cu-factor, num exc, num sig, num bg
|
---|
60 | # other info: zd th R750cor R750ref
|
---|
61 | #
|
---|
62 | # external (allow only 20min and nightly binning)
|
---|
63 | # --------
|
---|
64 | # time: time, delta time, start, stop
|
---|
65 | # flux: excrate, excerr
|
---|
66 | #
|
---|
67 | # collaborators
|
---|
68 | # -------------
|
---|
69 | # time: time, delta time, start, stop, ontime
|
---|
70 | # flux: excrate, excerr, corrate, corerr, flux, flux-err, significance
|
---|
71 | #
|
---|
72 | # additional information to put:
|
---|
73 | # ------------------------------
|
---|
74 | # timestamp of creation
|
---|
75 | # query (for debugging / answering questions)
|
---|
76 | # policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
|
---|
77 | #
|
---|
78 |
|
---|
79 |
|
---|
80 | function print_policy()
|
---|
81 | {
|
---|
82 | echo "#"
|
---|
83 | echo "# Data Usage Policy: "
|
---|
84 | if [ "$expert" == "no" ]
|
---|
85 | then
|
---|
86 | echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
|
---|
87 | echo "# paper (H. Anderhub et al. JINST 8 P6008) and the quick look analysis website "
|
---|
88 | echo "# (https://fact-project.org/monitoring)."
|
---|
89 | echo "#"
|
---|
90 | echo "# References: "
|
---|
91 | echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
|
---|
92 | echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
|
---|
93 | echo "# FACT Performance Paper: http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
|
---|
94 | echo "# http://iopscience.iop.org/1748-0221/9/10/P10012"
|
---|
95 | echo "# FACT quick look analysis: https://fact-project.org/monitoring"
|
---|
96 | echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
|
---|
97 | echo "# If you intend to use data or information from this website, please let us know for reference."
|
---|
98 | else
|
---|
99 | echo "# As a member or associated member of the FACT Collaboration, you have access to internal information."
|
---|
100 | echo "# Any publication using FACT internal information has to have the full FACT author list."
|
---|
101 | fi
|
---|
102 | echo "#"
|
---|
103 | }
|
---|
104 |
|
---|
105 | print_selection()
|
---|
106 | {
|
---|
107 | echo "#"
|
---|
108 | echo "# Your Selection: "
|
---|
109 | sourcename=`mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "SELECT fSourceName FROM Source WHERE fSourceKey="$source`
|
---|
110 | echo "# Source: "$sourcename
|
---|
111 | echo "# Time range: "$nightmin"-"$nightmax
|
---|
112 | echo "# Time format: "$timeunit
|
---|
113 | if [ $bin -lt 0 ]
|
---|
114 | then
|
---|
115 | unit="night(s)"
|
---|
116 | else
|
---|
117 | if [ $bin -eq 0 ]
|
---|
118 | then
|
---|
119 | unit="periods"
|
---|
120 | else
|
---|
121 | unit="minutes"
|
---|
122 | fi
|
---|
123 | fi
|
---|
124 | binning=`echo $bin | sed -e 's/-//'`" "$unit
|
---|
125 | echo "# Binning: "$binning
|
---|
126 | if [ "$expert" = "yes" ]
|
---|
127 | then
|
---|
128 | echo "# Additional Internal Selection: "
|
---|
129 | echo "# "`echo $table | sed -e 's/AnalysisResultsRun//'`"-Analysis was used."
|
---|
130 | if [ "$zdmax" != "" ]
|
---|
131 | then
|
---|
132 | echo "# Maximum Zenith Distance: "$zdmax" degree"
|
---|
133 | fi
|
---|
134 | if [ "$thmax" != "" ]
|
---|
135 | then
|
---|
136 | echo "# Maximum Trigger Threshold: "$thmax" DAC counts"
|
---|
137 | fi
|
---|
138 | if [ "$light" != "" ]
|
---|
139 | then
|
---|
140 | echo "# Light Condition Cut: "$lightcut
|
---|
141 | fi
|
---|
142 | if [ "$dust" != "" ]
|
---|
143 | then
|
---|
144 | echo "# Calima Cut: dust < "$dust" ug/cm3"
|
---|
145 | fi
|
---|
146 | if [ "$usedch" == "yes" ]
|
---|
147 | then
|
---|
148 | echo "# Data quality selection based on the cosmic ray rate was applied."
|
---|
149 | fi
|
---|
150 | fi
|
---|
151 | echo "#"
|
---|
152 | }
|
---|
153 |
|
---|
154 | function get_results()
|
---|
155 | {
|
---|
156 | # some query parts
|
---|
157 |
|
---|
158 | # some numbers for flux calculation
|
---|
159 | crabflux="3.37e-11"
|
---|
160 | fluxprec=13
|
---|
161 | crabflux="3.37"
|
---|
162 | fluxprec=2
|
---|
163 |
|
---|
164 | # some names and definitions needed several times below
|
---|
165 | # ontime
|
---|
166 | ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
|
---|
167 | ontime2=" fOnTimeAfterCuts "
|
---|
168 | ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
|
---|
169 | # zd and threshold
|
---|
170 | zenith="fZenithDistance"
|
---|
171 | thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
|
---|
172 | # correction factor for excess rate (formula by TB)
|
---|
173 | correvts=" fNumExcEvts*(pow(cos("$zenith"Mean*PI()/180),3)+14.8/21.9*pow(sin(2*"$zenith"Mean*PI()/180),5))/((1-0.00124/1.21*("$thresh"-500)*("$thresh">=500))) "
|
---|
174 | # conversion to CU (determined by DD for QLA)
|
---|
175 | # https://www.fact-project.org/logbook/showthread.php?tid=4927
|
---|
176 | cufactor="CUQLA(fNight)" # missing: ISDC analysis
|
---|
177 | # some calculations
|
---|
178 | excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
|
---|
179 | CU="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
|
---|
180 | CUerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
|
---|
181 | excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
|
---|
182 | CU2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
|
---|
183 | CUerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
|
---|
184 |
|
---|
185 | # columns to be selected
|
---|
186 | # for night-binning
|
---|
187 | ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
|
---|
188 | excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
|
---|
189 | significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
|
---|
190 | numexc="Sum(fNumExcEvts) AS numexc"
|
---|
191 | numsig="Sum(fNumSigEvts) AS numsig"
|
---|
192 | numbg="Sum(fNumBgEvts) AS numbg"
|
---|
193 | excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
|
---|
194 | correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
|
---|
195 | correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
|
---|
196 | cu=" ROUND("$CU", 2) AS cu"
|
---|
197 | cuerr=" ROUND("$CUerr", 2) AS cuerr"
|
---|
198 | flux="ROUND("$CU" * "$crabflux", 2) AS flux"
|
---|
199 | fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
|
---|
200 | # for minute binning
|
---|
201 | ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
|
---|
202 | excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
|
---|
203 | significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
|
---|
204 | numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
|
---|
205 | numsig2="Sum(o.sigevts) AS numsig"
|
---|
206 | numbg2="Sum(o.bgevts) AS numbg"
|
---|
207 | excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
|
---|
208 | correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
|
---|
209 | correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
|
---|
210 | cu2=" ROUND("$CU2", 2) AS cu"
|
---|
211 | cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
|
---|
212 | flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
|
---|
213 | fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
|
---|
214 |
|
---|
215 | case $timeunit in
|
---|
216 | mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
|
---|
217 | start=" Mjd(MIN(fRunStart)) AS start"
|
---|
218 | stop=" Mjd(MAX(fRunStop)) AS stop"
|
---|
219 | deltat=$delta" AS deltat"
|
---|
220 | time=" Mjd(MIN(fRunStart))+"$delta" AS time"
|
---|
221 | delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
|
---|
222 | start2=" Mjd(MIN(o.start)) AS start"
|
---|
223 | stop2=" Mjd(MAX(o.stop)) AS stop"
|
---|
224 | deltat2=$delta2" AS deltat"
|
---|
225 | time2=" Mjd(MIN(o.start))+"$delta2" AS time"
|
---|
226 | ;;
|
---|
227 | unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
|
---|
228 | start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
|
---|
229 | stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
|
---|
230 | deltat=$delta" AS deltat"
|
---|
231 | time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
|
---|
232 | delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
|
---|
233 | start2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
|
---|
234 | stop2=" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
|
---|
235 | deltat2=$delta2" AS deltat"
|
---|
236 | time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
|
---|
237 | ;;
|
---|
238 | *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
|
---|
239 | start=" MIN(fRunStart) AS start"
|
---|
240 | stop=" MAX(fRunStop) AS stop"
|
---|
241 | deltat=$delta" AS deltat"
|
---|
242 | time=" addtime(MIN(fRunStart), "$delta") AS time"
|
---|
243 | delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
|
---|
244 | start2=" MIN(o.start) AS start"
|
---|
245 | stop2=" MAX(o.stop) AS stop"
|
---|
246 | deltat2=$delta2" AS deltat"
|
---|
247 | time2=" addtime(MIN(o.start), "$delta2") AS time"
|
---|
248 | ;;
|
---|
249 | esac
|
---|
250 |
|
---|
251 | # from and join of query
|
---|
252 | from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
|
---|
253 |
|
---|
254 | # data check based on artificial trigger rate
|
---|
255 | # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
|
---|
256 | #dch=" AND fR750Cor/fR750Ref >0.93 "
|
---|
257 | dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
|
---|
258 |
|
---|
259 | # put together where-clause of query
|
---|
260 | # time range and source
|
---|
261 | where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
|
---|
262 | where=$where" AND NOT ISNULL(fNumExcEvts) "
|
---|
263 | # some sanity checks
|
---|
264 | where=$where" AND fRunTypeKey=1 "
|
---|
265 | # zd cut
|
---|
266 | if [ "$zdmax" != "" ]
|
---|
267 | then
|
---|
268 | where=$where" AND fZenithDistanceMax < "$zdmax
|
---|
269 | fi
|
---|
270 | # th cut
|
---|
271 | if [ "$thmax" != "" ]
|
---|
272 | then
|
---|
273 | where=$where" AND "$thresh" < "$thmax
|
---|
274 | fi
|
---|
275 | # dust cut
|
---|
276 | if [ "$dust" != "" ]
|
---|
277 | then
|
---|
278 | where=$where" AND fTNGDust<"$dust
|
---|
279 | fi
|
---|
280 | # light condition cut
|
---|
281 | if [ "$light" == "nomoon" ]
|
---|
282 | then
|
---|
283 | lightcut=" fZenithDistanceMoon>90"
|
---|
284 | fi
|
---|
285 | if [ "$light" == "dark" ]
|
---|
286 | then
|
---|
287 | lightcut=" fMoonZenithDistance>90 AND fSunZenithDistance>108 "
|
---|
288 | fi
|
---|
289 | if [ "$light" != "" ]
|
---|
290 | then
|
---|
291 | where=$where" AND "$lightcut
|
---|
292 | fi
|
---|
293 | querybase=$from$where
|
---|
294 |
|
---|
295 | if [ "$usedch" == "yes" ]
|
---|
296 | then
|
---|
297 | if [ "$dch" == "" ]
|
---|
298 | then
|
---|
299 | querydch=$dchstd
|
---|
300 | else
|
---|
301 | echo "you are using for datacheck: "$dch
|
---|
302 | querydch=$dch
|
---|
303 | fi
|
---|
304 | fi
|
---|
305 |
|
---|
306 |
|
---|
307 | if [ $bin -le 0 ]
|
---|
308 | then
|
---|
309 | # first part of the query
|
---|
310 | querystart="SELECT "
|
---|
311 | querystart=$querystart" "$time", "$start", "$stop", "
|
---|
312 | # final part of the query
|
---|
313 | if [ $bin -eq 0 ]
|
---|
314 | then
|
---|
315 | orderby=" fPeriod "
|
---|
316 | #querystart=$querystart" fPeriod AS num, "
|
---|
317 | queryend=" GROUP BY fPeriod "
|
---|
318 | else
|
---|
319 | num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
|
---|
320 | orderby=$num
|
---|
321 | #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
|
---|
322 | queryend=" GROUP BY "$num
|
---|
323 | fi
|
---|
324 | #queryend=" GROUP BY num "
|
---|
325 | if [ "$ontimelimit" = "" ]
|
---|
326 | then
|
---|
327 | queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
|
---|
328 | else
|
---|
329 | queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
|
---|
330 | fi
|
---|
331 |
|
---|
332 | # internal
|
---|
333 | queryint=$querystart
|
---|
334 | queryint=$queryint" "$excrate", "$correxcrate", "$cu", "$flux", "
|
---|
335 | queryint=$queryint" "$deltat", "$ontime", "
|
---|
336 | queryint=$queryint" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
|
---|
337 | queryint=$queryint" "$significance", "
|
---|
338 | queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
|
---|
339 | queryint=$queryint" "$numexc", "$numsig", "$numbg", "
|
---|
340 | queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
|
---|
341 | queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
|
---|
342 | queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
|
---|
343 | queryint=$queryint" "$querybase" "$querydch" "$queryend
|
---|
344 |
|
---|
345 | # for collaborators
|
---|
346 | querycol=$querystart
|
---|
347 | querycol=$querycol" "$excrate", "$correxcrate", "$cu", "$flux", "
|
---|
348 | querycol=$querycol" "$deltat", "$ontime", "
|
---|
349 | querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
|
---|
350 | querycol=$querycol" "$significance
|
---|
351 | querycol=$querycol" "$querybase" "$querydch" "$queryend
|
---|
352 |
|
---|
353 | # external
|
---|
354 | # no datacheck applied for external files
|
---|
355 | queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
|
---|
356 |
|
---|
357 | else
|
---|
358 | # first part of the query
|
---|
359 | querystart="SELECT "
|
---|
360 | querystart=$querystart" "$time2", "$start2", "$stop2", "
|
---|
361 |
|
---|
362 | # final part of the query
|
---|
363 | querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
|
---|
364 | querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
|
---|
365 | querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
|
---|
366 | querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
|
---|
367 | querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
|
---|
368 | querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
|
---|
369 | querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
|
---|
370 | querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
|
---|
371 |
|
---|
372 | # internal
|
---|
373 | queryint=$querystart
|
---|
374 | queryint=$queryint" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
|
---|
375 | queryint=$queryint" "$deltat2", "$ontime2", "
|
---|
376 | queryint=$queryint" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
|
---|
377 | queryint=$queryint" "$significance2", "
|
---|
378 | queryint=$queryint" avg(o.night) AS night, "
|
---|
379 | queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
|
---|
380 | queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
|
---|
381 | queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
|
---|
382 | queryint=$queryint" "$querybase
|
---|
383 |
|
---|
384 | # for collaborators
|
---|
385 | querycol=$querystart
|
---|
386 | querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
|
---|
387 | querycol=$querycol" "$deltat2", "$ontime2", "
|
---|
388 | querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
|
---|
389 | querycol=$querycol" "$significance2
|
---|
390 | querycol=$querycol" "$querybase
|
---|
391 |
|
---|
392 | # external
|
---|
393 | queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
|
---|
394 |
|
---|
395 | fi
|
---|
396 |
|
---|
397 | # write file for externals only for allowed binnings
|
---|
398 | if [ $bin -eq 20 ] || [ $bin -eq -1 ]
|
---|
399 | then
|
---|
400 | fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
|
---|
401 | if [ "$overwrite" = "yes" ]
|
---|
402 | then
|
---|
403 | if [ "$mode" != "auto" ]
|
---|
404 | then
|
---|
405 | echo "creating "$fileext" ..."
|
---|
406 | fi
|
---|
407 | echo "# This file was created at "`date` > $fileext
|
---|
408 | print_policy >> $fileext
|
---|
409 | fi
|
---|
410 | print_selection >> $fileext
|
---|
411 | headerext="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] (stop-start)/2["$timeunit"] excess-rate_error[evts/h] "
|
---|
412 | echo $headerext >> $fileext
|
---|
413 | #echo "$queryext"
|
---|
414 | mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryext" >> $fileext
|
---|
415 | #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryext"
|
---|
416 | fi
|
---|
417 | if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
|
---|
418 | then
|
---|
419 | return
|
---|
420 | fi
|
---|
421 |
|
---|
422 | fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
|
---|
423 | if [ "$overwrite" = "yes" ]
|
---|
424 | then
|
---|
425 | if [ "$mode" != "auto" ]
|
---|
426 | then
|
---|
427 | echo "creating "$fileint" ..."
|
---|
428 | fi
|
---|
429 | echo "# This file was created at "`date` > $fileint
|
---|
430 | print_policy >> $fileint
|
---|
431 | print_selection >> $fileint
|
---|
432 | echo "# The following query was used: " >> $fileint
|
---|
433 | echo "# "$queryint >> $fileint
|
---|
434 | echo "#" >> $fileint
|
---|
435 | fi
|
---|
436 | headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
|
---|
437 | headerint=$headerint" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
|
---|
438 | headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
|
---|
439 | echo $headerint >> $fileint
|
---|
440 | #echo "$queryint"
|
---|
441 | mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryint" >> $fileint
|
---|
442 | #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryint"
|
---|
443 | if [ "$mode" == "auto" ]
|
---|
444 | then
|
---|
445 | return
|
---|
446 | fi
|
---|
447 |
|
---|
448 | filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
|
---|
449 | if [ "$overwrite" = "yes" ]
|
---|
450 | then
|
---|
451 | echo "creating "$filecol" ..."
|
---|
452 | echo "# This file was created at "`date` > $filecol
|
---|
453 | print_policy >> $filecol
|
---|
454 | print_selection >> $filecol
|
---|
455 | echo "# The following query was used: " >> $filecol
|
---|
456 | echo "# "$querycol >> $filecol
|
---|
457 | echo "#" >> $filecol
|
---|
458 | fi
|
---|
459 | headercol="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
|
---|
460 | headercol=$headercol" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance "
|
---|
461 | echo $headercol >> $filecol
|
---|
462 | #echo "$querycol"
|
---|
463 | mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$querycol" >> $filecol
|
---|
464 | #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$querycol
|
---|
465 | }
|
---|
466 |
|
---|
467 | # evaluation of command line options (for usage with download.php)
|
---|
468 |
|
---|
469 | if [ ${#@} -eq 13 ]
|
---|
470 | then
|
---|
471 | #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
|
---|
472 | mode="auto"
|
---|
473 | overwrite="yes"
|
---|
474 | # setup
|
---|
475 | datapath="/home/factwww/dch/data"
|
---|
476 | #datapath="./data"
|
---|
477 | sqlpw=/home/fact/.mysql.pw
|
---|
478 | #sqlpw=/home/fact/.mysql.pw2
|
---|
479 | host=10.0.100.21
|
---|
480 | dbname=factdata
|
---|
481 | nightmin=$1
|
---|
482 | nightmax=$2
|
---|
483 | source=$3
|
---|
484 | bin=$4
|
---|
485 | if [ "$bin" == "00" ]
|
---|
486 | then
|
---|
487 | bin=0
|
---|
488 | fi
|
---|
489 | email=$5
|
---|
490 | table=$6
|
---|
491 | timeunit=$7
|
---|
492 | expert=$8
|
---|
493 | usedch=$9 # novalue gives same result as no
|
---|
494 | if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
|
---|
495 | then
|
---|
496 | zdmax=${10}
|
---|
497 | fi
|
---|
498 | if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
|
---|
499 | then
|
---|
500 | thmax=${11}
|
---|
501 | fi
|
---|
502 | if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
|
---|
503 | then
|
---|
504 | light=${12}
|
---|
505 | fi
|
---|
506 | if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
|
---|
507 | then
|
---|
508 | dust=${13}
|
---|
509 | fi
|
---|
510 | name=`echo $email | sed -e 's/@/-at-/'`
|
---|
511 | get_results
|
---|
512 |
|
---|
513 | # sending email
|
---|
514 | if [ "$expert" == "yes" ]
|
---|
515 | then
|
---|
516 | cat $fileint | mail -s 'FACT internal data download' -b qla@fact-project.org -r qla@fact-project.org $email
|
---|
517 | else
|
---|
518 | cat $fileext | mail -s 'FACT data download' -b qla@fact-project.org -r qla@fact-project.org $email
|
---|
519 | fi
|
---|
520 |
|
---|
521 | exit
|
---|
522 | fi
|
---|
523 |
|
---|
524 |
|
---|
525 |
|
---|
526 | # -------------------------------------------------------------------------------------- #
|
---|
527 | # SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
|
---|
528 | # -------------------------------------------------------------------------------------- #
|
---|
529 | # #
|
---|
530 | # The lines below define the basic setup for the database and give examples and #
|
---|
531 | # explanations for the various options available. #
|
---|
532 | # The request of the data itself is done with a smaller setup further down. #
|
---|
533 | # #
|
---|
534 | # -------------------------------------------------------------------------------------- #
|
---|
535 | #
|
---|
536 | # ----------
|
---|
537 | # DB SETUP
|
---|
538 | # ----------
|
---|
539 | # path to file with mysql password
|
---|
540 | sqlpw=/home/$USER/.mysql.pw
|
---|
541 | # host of mysql server with FACT DB
|
---|
542 | #host=lp-fact # ISDC
|
---|
543 | host=10.0.100.21 # LP or LP via vpn
|
---|
544 | #host=localhost # your local machine in case you have a copy of DB
|
---|
545 | # name of database
|
---|
546 | dbname=factdata
|
---|
547 | #
|
---|
548 | # -------------
|
---|
549 | # BASIC SETUP
|
---|
550 | # -------------
|
---|
551 | # output path
|
---|
552 | path=`dirname $0`
|
---|
553 | datapath=$path"/data"
|
---|
554 | # create directory for data files
|
---|
555 | if ! [ -e $datapath ]
|
---|
556 | then
|
---|
557 | mkdir $datapath
|
---|
558 | fi
|
---|
559 | # time unit
|
---|
560 | #timeunit=timestamp # default
|
---|
561 | #timeunit=unix
|
---|
562 | timeunit=mjd
|
---|
563 | # time binning
|
---|
564 | # positive values: minutes
|
---|
565 | # negative values: days
|
---|
566 | # special case 0: period
|
---|
567 | # for season binning choose -365 and according start date
|
---|
568 | #bin=20 # minutes
|
---|
569 | #bin=0 # period
|
---|
570 | bin=-1 # nightly
|
---|
571 | #bin=-365 # yearly
|
---|
572 | # choose analysis
|
---|
573 | #table="AnalysisResultsAllQLA" # N/A
|
---|
574 | table="AnalysisResultsRunLP" # QLA
|
---|
575 | #table="AnalysisResultsRunISDC" # ISDC
|
---|
576 | # time range
|
---|
577 | nightmin=20111115
|
---|
578 | nightmax=20201231
|
---|
579 | # overwrite dataset file?
|
---|
580 | # (useful to combine different binnings in one file -> set to "no")
|
---|
581 | overwrite="yes"
|
---|
582 | # optional: require minimal ontime per bin (default 20 min)
|
---|
583 | #ontimelimit=30 # 30 min
|
---|
584 | ontimelimit= # default 20 min
|
---|
585 | # data quality selection
|
---|
586 | # if you explicitely don't want a datacheck, you can comment the following line
|
---|
587 | usedch="yes"
|
---|
588 | # use your own datacheck instead
|
---|
589 | # use a line like the following defining your own data quality selection cut
|
---|
590 | #dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
|
---|
591 | # apply additional predefined cuts
|
---|
592 | # light conditions
|
---|
593 | #light="nomoon" # only data with no moon (but twilight allowed)
|
---|
594 | #light="dark" # only dark night data
|
---|
595 | # TNG dust - cut away data with calima
|
---|
596 | #dust=1
|
---|
597 | #dust=10
|
---|
598 |
|
---|
599 |
|
---|
600 | # -------------------------------------------------------------------------------------- #
|
---|
601 | # SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
|
---|
602 | # -------------------------------------------------------------------------------------- #
|
---|
603 | # #
|
---|
604 | # Adapt the lines below to your needs. #
|
---|
605 | # Overwrite default settings above. #
|
---|
606 | # The data-request is sent with the line 'get_results.' #
|
---|
607 | # Minumum setup: Define source key and name for file. #
|
---|
608 | # The list of source keys can be found at #
|
---|
609 | # https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
|
---|
610 | # More examples can be found further down. #
|
---|
611 | # #
|
---|
612 | # REMARKS: #
|
---|
613 | # - correction of effect of zd and threshold not yet finalized and only valid for QLA #
|
---|
614 | # - no CU-conversion available for ISDC-analysis so far (that for QLA is used instead) #
|
---|
615 | # #
|
---|
616 | # -------------------------------------------------------------------------------------- #
|
---|
617 |
|
---|
618 | # 501 MAGIC
|
---|
619 | source=2
|
---|
620 | name="Mrk501_2014_forMAGIC"
|
---|
621 | bin=-1
|
---|
622 | nightmin=20140714
|
---|
623 | nightmax=20140805
|
---|
624 | get_results
|
---|
625 |
|
---|
626 | bin=30
|
---|
627 | name="Mrk501_2014_forMAGIC30"
|
---|
628 | get_results
|
---|
629 |
|
---|
630 | bin=0
|
---|
631 | name="P"
|
---|
632 | nightmin=20140501
|
---|
633 | nightmax=20140930
|
---|
634 | get_results
|
---|
635 |
|
---|
636 | bin=20
|
---|
637 | nightmin=20140623
|
---|
638 | nightmax=20140623
|
---|
639 | name="Mrk501_test"
|
---|
640 | get_results
|
---|
641 |
|
---|
642 |
|
---|
643 | # end script here
|
---|
644 | exit
|
---|
645 |
|
---|
646 |
|
---|
647 |
|
---|
648 | #
|
---|
649 | # more examples
|
---|
650 | #
|
---|
651 |
|
---|
652 | # Mrk 421
|
---|
653 | source=1
|
---|
654 | name="Mrk421_nightly"
|
---|
655 | bin=-1
|
---|
656 | get_results
|
---|
657 | name="Mrk421_20min"
|
---|
658 | bin=20
|
---|
659 | get_results
|
---|
660 | name="Mrk421_3d"
|
---|
661 | bin=-3
|
---|
662 | get_results
|
---|
663 | name="Mrk421_10d"
|
---|
664 | bin=-10
|
---|
665 | get_results
|
---|
666 | name="Mrk421_period"
|
---|
667 | bin=0
|
---|
668 | get_results
|
---|
669 |
|
---|
670 |
|
---|
671 |
|
---|
672 | # Mrk 501
|
---|
673 | source=2
|
---|
674 | name="Mrk501_nightly"
|
---|
675 | bin=-1
|
---|
676 | get_results
|
---|
677 | name="Mrk501_20min"
|
---|
678 | bin=20
|
---|
679 | get_results
|
---|
680 | name="Mrk501_3d"
|
---|
681 | bin=-3
|
---|
682 | get_results
|
---|
683 | name="Mrk501_10d"
|
---|
684 | bin=-10
|
---|
685 | get_results
|
---|
686 | name="Mrk501_period"
|
---|
687 | bin=0
|
---|
688 | get_results
|
---|
689 |
|
---|
690 |
|
---|
691 |
|
---|
692 | # 2344
|
---|
693 | source=3
|
---|
694 | name="2344_nightly"
|
---|
695 | bin=-1
|
---|
696 | get_results
|
---|
697 | name="2344_20min"
|
---|
698 | bin=20
|
---|
699 | get_results
|
---|
700 | name="2344_period"
|
---|
701 | bin=0
|
---|
702 | get_results
|
---|
703 |
|
---|
704 |
|
---|
705 |
|
---|
706 | # 1959
|
---|
707 | source=7
|
---|
708 | name="1959_nightly"
|
---|
709 | bin=-1
|
---|
710 | get_results
|
---|
711 | name="1959_20min"
|
---|
712 | bin=20
|
---|
713 | get_results
|
---|
714 | name="1959_period"
|
---|
715 | bin=0
|
---|
716 | get_results
|
---|
717 |
|
---|
718 |
|
---|
719 |
|
---|
720 | # 0323
|
---|
721 | source=12
|
---|
722 | name="0323_nightly"
|
---|
723 | bin=-1
|
---|
724 | get_results
|
---|
725 | name="0323_20min"
|
---|
726 | bin=20
|
---|
727 | get_results
|
---|
728 | name="0323_period"
|
---|
729 | bin=0
|
---|
730 | get_results
|
---|
731 |
|
---|
732 |
|
---|
733 |
|
---|
734 | # crab
|
---|
735 | source=5
|
---|
736 | name="Crab_nightly"
|
---|
737 | bin=-1
|
---|
738 | get_results
|
---|
739 | name="Crab_20min"
|
---|
740 | bin=20
|
---|
741 | get_results
|
---|
742 | name="Crab_period"
|
---|
743 | bin=0
|
---|
744 | get_results
|
---|
745 | name="Crab_season"
|
---|
746 | bin=-365
|
---|
747 | nightmin=20110716
|
---|
748 | nightmax=20180716
|
---|
749 | get_results
|
---|
750 |
|
---|
751 |
|
---|
752 |
|
---|
753 | name="1959_2016"
|
---|
754 | source=7
|
---|
755 | bin=-1
|
---|
756 | nightmin=20160201
|
---|
757 | nightmax=20161105
|
---|
758 | get_results
|
---|
759 |
|
---|
760 | name="1959_all_variable"
|
---|
761 | overwrite="no"
|
---|
762 | source=7
|
---|
763 | bin=-365
|
---|
764 | nightmin=20120201
|
---|
765 | nightmax=20130131
|
---|
766 | get_results
|
---|
767 | nightmin=20130201
|
---|
768 | nightmax=20140131
|
---|
769 | get_results
|
---|
770 | nightmin=20140201
|
---|
771 | nightmax=20150131
|
---|
772 | get_results
|
---|
773 | bin=0
|
---|
774 | nightmin=20150201
|
---|
775 | nightmax=20160131
|
---|
776 | get_results
|
---|
777 | bin=-1
|
---|
778 | nightmin=20160201
|
---|
779 | nightmax=20170131
|
---|
780 | get_results
|
---|
781 | bin=0
|
---|
782 | nightmin=20170201
|
---|
783 | nightmax=20180131
|
---|
784 | get_results
|
---|
785 |
|
---|
786 |
|
---|
787 |
|
---|
788 | overwrite="yes"
|
---|
789 | name="1959_all_variable2"
|
---|
790 | overwrite="no"
|
---|
791 | source=7
|
---|
792 | bin=-365
|
---|
793 | nightmin=20120201
|
---|
794 | nightmax=20130131
|
---|
795 | get_results
|
---|
796 | nightmin=20130201
|
---|
797 | nightmax=20140131
|
---|
798 | get_results
|
---|
799 | nightmin=20140201
|
---|
800 | nightmax=20150131
|
---|
801 | get_results
|
---|
802 | bin=0
|
---|
803 | nightmin=20150201
|
---|
804 | nightmax=20160131
|
---|
805 | get_results
|
---|
806 | bin=-1
|
---|
807 | nightmin=20160201
|
---|
808 | nightmax=20160817
|
---|
809 | get_results
|
---|
810 | bin=0
|
---|
811 | nightmin=20160818
|
---|
812 | nightmax=20180131
|
---|
813 | get_results
|
---|
814 |
|
---|
815 |
|
---|
816 |
|
---|
817 | overwrite="yes"
|
---|
818 | bin=0
|
---|
819 | source=3
|
---|
820 | name="2344period"
|
---|
821 | get_results
|
---|
822 |
|
---|
823 |
|
---|
824 |
|
---|
825 | # flare night (HESS)
|
---|
826 | name="Mrk501_10min_flarenight"
|
---|
827 | source=2
|
---|
828 | bin=10
|
---|
829 | nightmin=20140623
|
---|
830 | nightmax=20140623
|
---|
831 | get_results
|
---|
832 |
|
---|
833 |
|
---|
834 |
|
---|
835 | # flare night (HESS)
|
---|
836 | name="Mrk501_5min_flarenight"
|
---|
837 | source=2
|
---|
838 | bin=5
|
---|
839 | nightmin=20140623
|
---|
840 | nightmax=20140623
|
---|
841 | get_results
|
---|
842 |
|
---|
843 |
|
---|
844 |
|
---|
845 |
|
---|
846 | # full sample
|
---|
847 | name="Mrk421_all_nightly"
|
---|
848 | source=1
|
---|
849 | get_results
|
---|
850 |
|
---|
851 | name="Mrk501_all_nightly"
|
---|
852 | source=2
|
---|
853 | get_results
|
---|
854 |
|
---|
855 | name="1959_all_nightly"
|
---|
856 | source=7
|
---|
857 | get_results
|
---|
858 |
|
---|
859 | name="2344_all_nightly"
|
---|
860 | source=3
|
---|
861 | get_results
|
---|
862 |
|
---|
863 |
|
---|
864 |
|
---|
865 | name="HESE20160427"
|
---|
866 | source=19
|
---|
867 | nightmin=20160425
|
---|
868 | bin=-10
|
---|
869 | get_results
|
---|
870 |
|
---|
871 | name="AMON20160731"
|
---|
872 | source=21
|
---|
873 | nightmin=20160730
|
---|
874 | bin=-10
|
---|
875 | get_results
|
---|
876 |
|
---|
877 |
|
---|
878 |
|
---|