source: trunk/DataCheck/Tools/get_data.sh@ 19568

Last change on this file since 19568 was 19568, checked in by Daniela Dorner, 5 years ago
added uncorrected CU for internal files
  • Property svn:executable set to *
File size: 32.5 KB
Line 
1#!/bin/bash
2
3# ---------------------------------------------------------------- #
4# README README README README README README README README README #
5# ---------------------------------------------------------------- #
6# #
7# To use this script, you need #
8# - a computer with access to the FACT database in La Palma #
9# - a file with the password of a valid mysql-user #
10# - to define the setup below for #
11# a) the DB access #
12# b) the data you want to have #
13# #
14# To define the setup, search for SETUP in this script and #
15# read the details there #
16# #
17# Per data request, you get up to 3 files: #
18# *_internal.dat #
19# *_collaborators.dat #
20# *_external.dat (only if binning is 20min or nightly) #
21# #
22# Please have in mind that this started as a tool for myself, then #
23# others started using it. Also the script is not yet finalized. #
24# In case you find problems and/or have a feature request, please #
25# send and email to dorner@astro.uni-wuerzburg.de #
26# #
27# ---------------------------------------------------------------- #
28# README README README README README README README README README #
29# ---------------------------------------------------------------- #
30
31
32
33# ToDo (notes DD):
34# ----------------
35# - do conversion to fluxes using FACT Crab spectrum
36# - add < 20121212 data for QLA
37# - new version of zd/th-correction and CU from Aachen when ready
38# nice-to-have
39# - add E2dNdE
40# - functionality to determine start time for seaon-binning
41# - file as attachment to email for download.php
42
43#
44# content of files (wish list):
45# -----------------------------
46# REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
47#
48# internal
49# --------
50# time: time, delta time, start, stop, ontime
51# flux: excrate, excerr, CU, CUerr, corrate, corerr, CUcor CUcorerr, fluxcor, fluxcorerr,
52# other info on flux: signif, cu-factor, num exc, num sig, num bg
53# other info: zd th R750cor R750ref
54#
55# external (allow only 20min and nightly binning)
56# --------
57# time: time, delta time, start, stop
58# flux: excrate, excerr
59#
60# collaborators
61# -------------
62# time: time, delta time, start, stop, ontime
63# flux: excrate, excerr, corrate, corerr, CUcor CUcorerr, fluxcor, fluxcorrerr, significance
64#
65# additional information to put:
66# ------------------------------
67# timestamp of creation
68# query (for debugging / answering questions)
69# policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
70#
71
72
73function print_policy()
74{
75 echo "#"
76 echo "# Data Usage Policy: "
77 if [ "$expert" == "no" ]
78 then
79 echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
80 echo "# paper (H. Anderhub et al. JINST 8 P6008) and the quick look analysis website "
81 echo "# (https://fact-project.org/monitoring)."
82 echo "#"
83 echo "# References: "
84 echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
85 echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
86 echo "# FACT Performance Paper: http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
87 echo "# http://iopscience.iop.org/1748-0221/9/10/P10012"
88 echo "# FACT quick look analysis: https://fact-project.org/monitoring"
89 echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
90 echo "# If you intend to use data or information from this website, please let us know for reference."
91 else
92 echo "# As a member, associated member or collaborator of the FACT Collaboration, you have access to internal information."
93 echo "# Any publication using FACT internal information has to have the full FACT author list."
94 fi
95 echo "#"
96}
97
98print_selection()
99{
100 echo "#"
101 echo "# Your Selection: "
102 sourcename=`mysql --defaults-file=$sqlpw -s -e "SELECT fSourceName FROM Source WHERE fSourceKey="$source`
103 echo "# Source: "$sourcename
104 echo "# Time range: "$nightmin"-"$nightmax
105 echo "# Time format: "$timeunit
106 if [ $bin -lt 0 ]
107 then
108 unit="night(s)"
109 else
110 if [ $bin -eq 0 ]
111 then
112 unit="periods"
113 else
114 unit="minutes"
115 fi
116 fi
117 binning=`echo $bin | sed -e 's/-//'`" "$unit
118 echo "# Binning: "$binning
119 if [ "$expert" != "no" ]
120 then
121 echo "# Additional Internal Selection: "
122 echo "# "`echo $table | sed -e 's/AnalysisResultsRun//'`"-Analysis was used."
123 if [ "$zdmax" != "" ]
124 then
125 echo "# Maximum Zenith Distance: "$zdmax" degree"
126 fi
127 if [ "$thmax" != "" ]
128 then
129 echo "# Maximum Trigger Threshold: "$thmax" DAC counts"
130 fi
131 if [ "$light" != "" ]
132 then
133 echo "# Light Condition Cut: "$lightcut
134 fi
135 if [ "$dust" != "" ]
136 then
137 echo "# Calima Cut: dust < "$dust" ug/cm3"
138 fi
139 if [ "$usedch" == "yes" ]
140 then
141 echo "# Data quality selection was applied. "
142 if [ "$dch" == "" ]
143 then
144 echo "# Standard data check based on cosmic-ray rate: "$dchstd
145 else
146 echo "# You selected a custom datacheck: "$dch
147 fi
148 fi
149 fi
150 echo "#"
151}
152
153function get_results()
154{
155 # some query parts
156
157 # these values use the Crab spectrum from HESS 2006
158 # hess 2006 http://www.aanda.org/articles/aa/pdf/2006/39/aa5351-06.pdf
159 # PL.SetParameter(1,-2.63);
160 # PL.SetParameter(0, 3.45e-11);
161 case "$table" in
162 "AnalysisResultsRunISDC")
163 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.2)"
164 thfactor="(1.18-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00062)"
165 # ETh: 1070 GeV
166 crabflux="1.89"
167 ;;
168 "AnalysisResultsRunCutsLC")
169 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.5)"
170 thfactor="(1.37-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00118)"
171 # ETh: 785 GeV
172 crabflux="3.14"
173 ;;
174 "AnalysisResultsRunLP")
175 zdfactor="(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))"
176 #zdfactor="(1/(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5)))"
177 thfactor="(1-0.00124/1.21*(fThresholdMinSet-500)*(fThresholdMinSet>=500))"
178 # ETh: 1100 GeV
179 crabflux="1.81"
180 ;;
181 "*")
182 zdfactor="0"
183 thfactor="0"
184 crabflux="0"
185 ;;
186 esac
187
188 # conversion from crab units to fluxes (if a different value is given in setup
189 if [ "$crabfluxconv" != "" ]
190 then
191 crabflux=$crabfluxconv #e-11
192 fi
193 #echo "crabflux: "$crabflux
194 fluxprec=2
195
196 # some names and definitions needed several times below
197 # ontime
198 ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
199 ontime2=" fOnTimeAfterCuts "
200 ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
201 # zd and threshold
202 zenith="fZenithDistance"
203 thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
204 # correction for zd- and th-dependence
205 correvts=" fNumExcEvts/"$zdfactor"/"$thfactor
206 # conversion to CU - stored in each result-table (determined by DD Feb 2019)
207 cufactor="fCU"
208 # some calculations
209 excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
210 CU="SUM(fNumExcEvts/"$cufactor")/SUM("$ontimeif")*3600"
211 CUerr=$excerr"/SUM("$ontimeif")*3600*SUM(fNumExcEvts/"$cufactor")/SUM(fNumExcEvts)"
212 CUcor="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
213 CUcorerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
214 excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
215 CU2="SUM((o.sigevts-o.bgevts)/o.cufactor)/SUM(o.ot)*3600"
216 CUerr2=$excerr2"/SUM(o.ot)*3600*SUM((o.sigevts-o.bgevts)/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
217 CUcor2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
218 CUcorerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
219
220 # columns to be selected
221 # for night-binning
222 ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
223 excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
224 significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
225 numexc="Sum(fNumExcEvts) AS numexc"
226 numsig="Sum(fNumSigEvts) AS numsig"
227 numbg="Sum(fNumBgEvts) AS numbg"
228 excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
229 correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
230 correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
231 cu=" ROUND("$CU", 2) AS cu"
232 cuerr=" ROUND("$CUerr", 2) AS cuerr"
233 cucor=" ROUND("$CUcor", 2) AS cu"
234 cucorerr=" ROUND("$CUcorerr", 2) AS cuerr"
235 flux="ROUND("$CU" * "$crabflux", 2) AS flux"
236 fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
237 # for minute binning
238 ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
239 excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
240 significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
241 numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
242 numsig2="Sum(o.sigevts) AS numsig"
243 numbg2="Sum(o.bgevts) AS numbg"
244 excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
245 correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
246 correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
247 cu2=" ROUND("$CU2", 2) AS cu"
248 cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
249 cucor2=" ROUND("$CUcor2", 2) AS cu"
250 cucorerr2=" ROUND("$CUcorerr2", 2) AS cuerr"
251 flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
252 fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
253
254 case $timeunit in
255 mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
256 start=" Mjd(MIN(fRunStart)) AS start"
257 stop=" Mjd(MAX(fRunStop)) AS stop"
258 deltat=$delta" AS deltat"
259 time=" Mjd(MIN(fRunStart))+"$delta" AS time"
260 delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
261 start2=" Mjd(MIN(o.start)) AS start"
262 stop2=" Mjd(MAX(o.stop)) AS stop"
263 deltat2=$delta2" AS deltat"
264 time2=" Mjd(MIN(o.start))+"$delta2" AS time"
265 ;;
266 unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
267 start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
268 stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
269 deltat=$delta" AS deltat"
270 time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
271 delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
272 start2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
273 stop2=" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
274 deltat2=$delta2" AS deltat"
275 time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
276 ;;
277 *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
278 start=" MIN(fRunStart) AS start"
279 stop=" MAX(fRunStop) AS stop"
280 deltat=$delta" AS deltat"
281 time=" addtime(MIN(fRunStart), "$delta") AS time"
282 delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
283 start2=" MIN(o.start) AS start"
284 stop2=" MAX(o.stop) AS stop"
285 deltat2=$delta2" AS deltat"
286 time2=" addtime(MIN(o.start), "$delta2") AS time"
287 ;;
288 esac
289
290 # from and join of query
291 from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
292
293 # data check based on artificial trigger rate
294 # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
295 #dch=" AND fR750Cor/fR750Ref >0.93 "
296 dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
297
298 # put together where-clause of query
299 # time range and source
300 where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
301 where=$where" AND NOT ISNULL(fNumExcEvts) "
302 # some sanity checks
303 where=$where" AND fRunTypeKey=1 "
304 # zd cut
305 if [ "$zdmax" != "" ]
306 then
307 where=$where" AND fZenithDistanceMax < "$zdmax
308 fi
309 # th cut
310 if [ "$thmax" != "" ]
311 then
312 where=$where" AND "$thresh" < "$thmax
313 fi
314 # dust cut
315 if [ "$dust" != "" ]
316 then
317 where=$where" AND fTNGDust<"$dust
318 fi
319 # light condition cut
320 if [ "$light" == "nomoon" ]
321 then
322 lightcut=" fZenithDistanceMoon>90"
323 fi
324 if [ "$light" == "dark" ]
325 then
326 lightcut=" fMoonZenithDistance>90 AND fSunZenithDistance>108 "
327 fi
328 if [ "$light" != "" ]
329 then
330 where=$where" AND "$lightcut
331 fi
332 querybase=$from$where
333
334 if [ "$usedch" == "yes" ]
335 then
336 if [ "$dch" == "" ]
337 then
338 querydch=$dchstd
339 else
340 echo "you are using for datacheck: "$dch
341 querydch=$dch
342 fi
343 else
344 querydch=
345 fi
346
347
348 if [ $bin -le 0 ]
349 then
350 # first part of the query
351 querystart="SELECT "
352 querystart=$querystart" "$time", "$start", "$stop", "
353 # final part of the query
354 if [ $bin -eq 0 ]
355 then
356 orderby=" fPeriod "
357 #querystart=$querystart" fPeriod AS num, "
358 queryend=" GROUP BY fPeriod "
359 else
360 num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
361 orderby=$num
362 #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
363 queryend=" GROUP BY "$num
364 fi
365 #queryend=" GROUP BY num "
366 if [ "$ontimelimit" = "" ]
367 then
368 queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
369 else
370 queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
371 fi
372
373 # internal
374 queryint=$querystart
375 queryint=$queryint" "$excrate", "$cu", "$correxcrate", "$cucor", "$flux", "
376 queryint=$queryint" "$deltat", "$ontime", "
377 queryint=$queryint" "$excrateerr", "$cuerr", "$correxcrateerr", "$cucorerr", "$fluxerr", "
378 queryint=$queryint" "$significance", "
379 queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
380 queryint=$queryint" "$numexc", "$numsig", "$numbg", "
381 queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
382 queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
383 queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
384 queryint=$queryint" "$querybase" "$querydch" "$queryend
385
386 # for collaborators
387 querycol=$querystart
388 querycol=$querycol" "$excrate", "$correxcrate", "$cucor", "$flux", "
389 querycol=$querycol" "$deltat", "$ontime", "
390 querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cucorerr", "$fluxerr", "
391 querycol=$querycol" "$significance
392 querycol=$querycol" "$querybase" "$querydch" "$queryend
393
394 # external
395 # no datacheck applied for external files
396 queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
397
398 else
399 # first part of the query
400 querystart="SELECT "
401 querystart=$querystart" "$time2", "$start2", "$stop2", "
402
403 # final part of the query
404 querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
405 querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
406 querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
407 querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
408 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
409 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
410 querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
411 querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
412
413 # internal
414 queryint=$querystart
415 queryint=$queryint" "$excrate2", "$cu2", "$correxcrate2", "$cucor2", "$flux2", "
416 queryint=$queryint" "$deltat2", "$ontime2", "
417 queryint=$queryint" "$excrateerr2", "$cuerr2", "$correxcrateerr2", "$cucorerr2", "$fluxerr2", "
418 queryint=$queryint" "$significance2", "
419 queryint=$queryint" avg(o.night) AS night, "
420 queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
421 queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
422 queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
423 queryint=$queryint" "$querybase
424
425 # for collaborators
426 querycol=$querystart
427 querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
428 querycol=$querycol" "$deltat2", "$ontime2", "
429 querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
430 querycol=$querycol" "$significance2
431 querycol=$querycol" "$querybase
432
433 # external
434 queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
435
436 fi
437
438 # write file for externals only for allowed binnings
439 if [ $bin -eq 20 ] || [ $bin -eq -1 ]
440 then
441 fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
442 if [ "$overwrite" = "yes" ]
443 then
444 if [ "$mode" != "auto" ]
445 then
446 echo "creating "$fileext" ..."
447 fi
448 echo "# This file was created at "`date` > $fileext
449 print_policy >> $fileext
450 fi
451 print_selection >> $fileext
452 headerext="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] (stop-start)/2["$timeunit"] excess-rate_error[evts/h] "
453 echo $headerext >> $fileext
454 #echo "$queryext"
455 mysql --defaults-file=$sqlpw -s -e "$queryext" >> $fileext
456 #mysql --defaults-file=$sqlpw -e "$queryext"
457 fi
458 if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
459 then
460 return
461 fi
462
463 fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
464 if [ "$overwrite" = "yes" ]
465 then
466 if [ "$mode" != "auto" ]
467 then
468 echo "creating "$fileint" ..."
469 fi
470 echo "# This file was created at "`date` > $fileint
471 print_policy >> $fileint
472 fi
473 print_selection >> $fileint
474 echo "# The following query was used: " >> $fileint
475 echo "# $queryint" >> $fileint
476 echo "#" >> $fileint
477 headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] flux[CU] corrected_excess-rate[evts/h] corrected_flux[CU] corrected_flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
478 headerint=$headerint" excess-rate_error[evts/h] flux_error[CU] corrected_excess-rate_error[evts/h] corrected_flux_error[CU] corrected_flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
479 headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
480 echo $headerint >> $fileint
481 #echo "$queryint"
482 mysql --defaults-file=$sqlpw -s -e "$queryint" >> $fileint
483 #mysql --defaults-file=$sqlpw -e "$queryint"
484 if [ "$mode" == "auto" ]
485 then
486 return
487 fi
488
489 filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
490 if [ "$overwrite" = "yes" ]
491 then
492 echo "creating "$filecol" ..."
493 echo "# This file was created at "`date` > $filecol
494 print_policy >> $filecol
495 fi
496 print_selection >> $filecol
497 #echo "# The following query was used: " >> $filecol
498 #echo "# $querycol" >> $filecol
499 #echo "#" >> $filecol
500 headercol="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] corrected_flux[CU] corrected_flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
501 headercol=$headercol" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] corrected_flux_error[CU] corrected_flux_error[e-11/cm2/s] significance "
502 echo $headercol >> $filecol
503 #echo "$querycol"
504 mysql --defaults-file=$sqlpw -s -e "$querycol" >> $filecol
505 #mysql --defaults-file=$sqlpw -e "$querycol
506}
507
508# evaluation of command line options (for usage with download.php)
509
510if [ ${#@} -eq 13 ]
511then
512 #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
513 mode="auto"
514 overwrite="yes"
515 # setup for usage with download.php
516 datapath="/home/factwww/dch/data"
517 #datapath="./data"
518 sqlpw=/home/fact/.mysql.pw
519 #sqlpw=/home/fact/.mysql.pw2
520 #host=10.0.100.21
521 #dbname=factdata
522 nightmin=$1
523 nightmax=$2
524 source=$3
525 bin=$4
526 if [ "$bin" == "00" ]
527 then
528 bin=0
529 fi
530 email=$5
531 table=$6
532 timeunit=$7
533 expert=$8
534 usedch=$9 # novalue gives same result as no
535 if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
536 then
537 zdmax=${10}
538 fi
539 if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
540 then
541 thmax=${11}
542 fi
543 if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
544 then
545 light=${12}
546 fi
547 if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
548 then
549 dust=${13}
550 fi
551 name=`echo $email | sed -e 's/@/-at-/'`
552 get_results
553
554 # sending email
555 if [ "$expert" == "yes" ]
556 then
557 cat $fileint | mail -s 'FACT internal data download' -b qla@fact-project.org -r qla@fact-project.org $email
558 else
559 cat $fileext | mail -s 'FACT data download' -b qla@fact-project.org -r qla@fact-project.org $email
560 fi
561
562 exit
563fi
564
565
566
567# -------------------------------------------------------------------------------------- #
568# SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
569# -------------------------------------------------------------------------------------- #
570# #
571# The lines below define the basic setup for the database and give examples and #
572# explanations for the various options available. #
573# The request of the data itself is done with a smaller setup further down. #
574# #
575# -------------------------------------------------------------------------------------- #
576#
577# ----------
578# DB SETUP
579# ----------
580# path to file with mysql setup (user, password, host, db-name)
581sqlpw=/home/$USER/.mysql.pw.local
582sqlpw=/home/$USER/.mysql.pw.ethz.fact
583#
584# -------------
585# BASIC SETUP
586# -------------
587# output path
588path=`dirname $0`
589datapath=$path"/data"
590# create directory for data files
591if ! [ -e $datapath ]
592then
593 mkdir $datapath
594fi
595# time unit
596#timeunit=timestamp # default
597#timeunit=unix
598timeunit=mjd
599# time binning
600# positive values: minutes
601# negative values: days
602# special case 0: period
603# for season binning choose -365 and according start date
604#bin=20 # minutes
605#bin=0 # period
606bin=-1 # nightly
607#bin=-365 # yearly
608# choose analysis
609#table="AnalysisResultsAllQLA" # N/A
610#table="AnalysisResultsRunLP" # QLA
611#table="AnalysisResultsRunISDC" # ISDC
612table="AnalysisResultsRunCutsLC" # CutsLC
613# time range
614nightmin=20111115
615nightmax=20201231
616# overwrite dataset file?
617# (useful to combine different binnings in one file -> set to "no")
618overwrite="yes"
619# optional: require minimal ontime per bin (default 20 min)
620#ontimelimit=30 # 30 min
621ontimelimit= # default 20 min
622# data quality selection
623# if you explicitely don't want a datacheck, you can comment the following line
624usedch="yes"
625# use your own datacheck instead
626# use a line like the following defining your own data quality selection cut
627#dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
628# apply additional predefined cuts
629# light conditions
630#light="nomoon" # only data with no moon (but twilight allowed)
631#light="dark" # only dark night data
632# TNG dust - cut away data with calima
633#dust=1
634#dust=10
635# use different conversion from CU to fluxes
636#crabfluxconv="2.5"
637
638
639# -------------------------------------------------------------------------------------- #
640# SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
641# -------------------------------------------------------------------------------------- #
642# #
643# Adapt the lines below to your needs. #
644# Overwrite default settings above. #
645# The data-request is sent with the line 'get_results'. #
646# Minumum setup: Define source key and name for file. #
647# The list of source keys can be found at #
648# https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
649# More examples can be found further down. #
650# #
651# -------------------------------------------------------------------------------------- #
652
653# Mrk 501
654# Mrk 501
655source=2
656zdmax=90
657thmax=1500
658# new analysis
659table="AnalysisResultsRunCutsLC" # CutsLC
660nightmin=20140520
661nightmax=20140930
662name="Mrk501_nightly_newAnalysis_forHESS"
663bin=-1
664get_results
665name="Mrk501_7d_newAnalysis_forHESS"
666bin=-7
667get_results
668nightmin=20140623
669nightmax=20140623
670name="Mrk501_5min_FlareNight_newAnalysis_forHESS"
671bin=5
672get_results
673nightmin=20140622
674nightmax=20140624
675name="Mrk501_5min_Flare_newAnalysis_forHESS"
676bin=5
677get_results
678
679exit
680
681# LC for ICRC
682table="AnalysisResultsRunCutsLC" # CutsLC
683bin=-1
684# Mrk 421
685source=1
686name="Mrk421_all_nightly"
687get_results
688# Mrk 501
689source=2
690name="Mrk501_all_nightly"
691get_results
692# 1959
693source=7
694name="1959_all_nightly"
695get_results
696
697
698exit
699
700# LC for INTEGRAL Proposal
701table="AnalysisResultsRunCutsLC" # CutsLC
702bin=-1
703# Mrk 421
704source=1
705name="Mrk421_all_nightly"
706get_results
707# Mrk 501
708source=2
709name="Mrk501_all_nightly"
710get_results
711# 1959
712source=7
713name="1959_all_nightly"
714get_results
715# for other INTEGRAL proposal
716# Mrk 421
717bin=0
718source=1
719name="Mrk421_all_period"
720get_results
721
722exit
723
724
725# Crab
726source=5
727nightmin=20121214
728nightmax=20180418
729table="AnalysisResultsRunCutsLC" # CutsLC
730bin=-1
731zdmax=30
732thmax=330
733name="testCrabnightlycheck"
734get_results
735
736exit
737
738# Crab
739source=5
740nightmin=20121214
741nightmax=20180418
742table="AnalysisResultsRunCutsLC" # CutsLC
743bin=20
744name="testCrab20min"
745get_results
746
747exit
748
749# Crab
750source=5
751nightmin=20121214
752nightmax=20180418
753table="AnalysisResultsRunCutsLC" # CutsLC
754bin=-1
755name="testCrabnightly"
756get_results
757
758exit
759
760# Crab
761source=5
762#nightmin=20121214
763#nightmax=20180418
764table="AnalysisResultsRunCutsLC" # CutsLC
765bin=-1
766name="testCraball"
767get_results
768
769exit
770
771
772# Crab
773source=5
774nightmin=20121214
775nightmax=20180418
776table="AnalysisResultsRunCutsLC" # CutsLC
777bin=60
778name="testCrab"
779get_results
780
781exit
782
783
784# for 2344 paper with MAGIC
785
786source=3
787zdmax=90
788thmax=1500
789
790# new analysis
791table="AnalysisResultsRunCutsLC" # CutsLC
792nightmin=20160618
793nightmax=20160815
794name="2344_2016flare_newAnalysis"
795bin=-7
796get_results
797overwrite="no"
798bin=-28
799nightmin=20160816
800nightmax=20161031
801get_results
802overwrite="yes"
803
804# ISDC analysis
805table="AnalysisResultsRunISDC"
806nightmin=20160618
807nightmax=20160815
808name="2344_2016flare_stdAnalysis"
809bin=-7
810get_results
811overwrite="no"
812bin=-28
813nightmin=20160816
814nightmax=20161031
815get_results
816overwrite="yes"
817
818
819usedch="no"
820# new analysis
821table="AnalysisResultsRunCutsLC" # CutsLC
822nightmin=20160618
823nightmax=20160815
824name="2344_2016flare_newAnalysis_noDCh"
825bin=-7
826get_results
827overwrite="no"
828bin=-28
829nightmin=20160816
830nightmax=20161031
831get_results
832overwrite="yes"
833
834# ISDC analysis
835table="AnalysisResultsRunISDC"
836nightmin=20160618
837nightmax=20160815
838name="2344_2016flare_stdAnalysis_noDCh"
839bin=-7
840get_results
841overwrite="no"
842bin=-28
843nightmin=20160816
844nightmax=20161031
845get_results
846overwrite="yes"
847
848usedch="yes"
849
850
851exit
852
853# Mrk 501
854source=2
855zdmax=90
856thmax=1500
857# new analysis
858table="AnalysisResultsRunCutsLC" # CutsLC
859nightmin=20140520
860nightmax=20140930
861name="Mrk501_nightly_newAnalysis_forHESS"
862bin=-1
863get_results
864name="Mrk501_7d_newAnalysis_forHESS"
865bin=-7
866get_results
867nightmin=20140623
868nightmax=20140623
869name="Mrk501_5min_FlareNight_newAnalysis_forHESS"
870bin=5
871get_results
872# isdc analysis
873thmax=850
874table="AnalysisResultsRunISDC"
875nightmin=20140520
876nightmax=20140930
877name="Mrk501_nightly_stdAnalysis_forHESS"
878bin=-1
879get_results
880name="Mrk501_7d_stdAnalysis_forHESS"
881bin=-7
882get_results
883nightmin=20140623
884nightmax=20140623
885name="Mrk501_5min_FlareNight_stdAnalysis_forHESS"
886bin=5
887get_results
888# qla
889thmax=500
890table="AnalysisResultsRunLP"
891nightmin=20140520
892nightmax=20140930
893name="Mrk501_nightly_QLA_forHESS"
894bin=-1
895get_results
896
897exit
898
899# 501 MAGIC
900source=2
901name="Mrk501_2014_forMAGIC"
902bin=-1
903crabfluxconv="2.87" # using threshold of 830GeV
904crabfluxconv="3.01" # using threshold of 830GeV and MAGIC Crab spectrum
905nightmin=20140714
906nightmax=20140805
907get_results
908
909name="Mrk501_2014_forMAGIC_20min"
910bin=20
911get_results
912
913
914exit
915
916bin=30
917name="Mrk501_2014_forMAGIC30"
918get_results
919
920bin=0
921name="P"
922nightmin=20140501
923nightmax=20140930
924get_results
925
926bin=20
927nightmin=20140623
928nightmax=20140623
929name="Mrk501_test"
930get_results
931
932
933# end script here
934exit
935
936
937
938#
939# more examples
940#
941
942# Mrk 421
943source=1
944name="Mrk421_nightly"
945bin=-1
946get_results
947name="Mrk421_20min"
948bin=20
949get_results
950name="Mrk421_3d"
951bin=-3
952get_results
953name="Mrk421_10d"
954bin=-10
955get_results
956name="Mrk421_period"
957bin=0
958get_results
959
960
961
962# Mrk 501
963source=2
964name="Mrk501_nightly"
965bin=-1
966get_results
967name="Mrk501_20min"
968bin=20
969get_results
970name="Mrk501_3d"
971bin=-3
972get_results
973name="Mrk501_10d"
974bin=-10
975get_results
976name="Mrk501_period"
977bin=0
978get_results
979
980
981
982# 2344
983source=3
984name="2344_nightly"
985bin=-1
986get_results
987name="2344_20min"
988bin=20
989get_results
990name="2344_period"
991bin=0
992get_results
993
994
995
996# 1959
997source=7
998name="1959_nightly"
999bin=-1
1000get_results
1001name="1959_20min"
1002bin=20
1003get_results
1004name="1959_period"
1005bin=0
1006get_results
1007
1008
1009
1010# 0323
1011source=12
1012name="0323_nightly"
1013bin=-1
1014get_results
1015name="0323_20min"
1016bin=20
1017get_results
1018name="0323_period"
1019bin=0
1020get_results
1021
1022
1023
1024# crab
1025source=5
1026name="Crab_nightly"
1027bin=-1
1028get_results
1029name="Crab_20min"
1030bin=20
1031get_results
1032name="Crab_period"
1033bin=0
1034get_results
1035name="Crab_season"
1036bin=-365
1037nightmin=20110716
1038nightmax=20180716
1039get_results
1040
1041
1042
1043name="1959_2016"
1044source=7
1045bin=-1
1046nightmin=20160201
1047nightmax=20161105
1048get_results
1049
1050name="1959_all_variable"
1051overwrite="no"
1052source=7
1053bin=-365
1054nightmin=20120201
1055nightmax=20130131
1056get_results
1057nightmin=20130201
1058nightmax=20140131
1059get_results
1060nightmin=20140201
1061nightmax=20150131
1062get_results
1063bin=0
1064nightmin=20150201
1065nightmax=20160131
1066get_results
1067bin=-1
1068nightmin=20160201
1069nightmax=20170131
1070get_results
1071bin=0
1072nightmin=20170201
1073nightmax=20180131
1074get_results
1075
1076
1077
1078overwrite="yes"
1079name="1959_all_variable2"
1080overwrite="no"
1081source=7
1082bin=-365
1083nightmin=20120201
1084nightmax=20130131
1085get_results
1086nightmin=20130201
1087nightmax=20140131
1088get_results
1089nightmin=20140201
1090nightmax=20150131
1091get_results
1092bin=0
1093nightmin=20150201
1094nightmax=20160131
1095get_results
1096bin=-1
1097nightmin=20160201
1098nightmax=20160817
1099get_results
1100bin=0
1101nightmin=20160818
1102nightmax=20180131
1103get_results
1104
1105
1106
1107overwrite="yes"
1108bin=0
1109source=3
1110name="2344period"
1111get_results
1112
1113
1114
1115# flare night (HESS)
1116name="Mrk501_10min_flarenight"
1117source=2
1118bin=10
1119nightmin=20140623
1120nightmax=20140623
1121get_results
1122
1123
1124
1125# flare night (HESS)
1126name="Mrk501_5min_flarenight"
1127source=2
1128bin=5
1129nightmin=20140623
1130nightmax=20140623
1131get_results
1132
1133
1134
1135
1136# full sample
1137name="Mrk421_all_nightly"
1138source=1
1139get_results
1140
1141name="Mrk501_all_nightly"
1142source=2
1143get_results
1144
1145name="1959_all_nightly"
1146source=7
1147get_results
1148
1149name="2344_all_nightly"
1150source=3
1151get_results
1152
1153
1154
1155name="HESE20160427"
1156source=19
1157nightmin=20160425
1158bin=-10
1159get_results
1160
1161name="AMON20160731"
1162source=21
1163nightmin=20160730
1164bin=-10
1165get_results
1166
1167
1168
Note: See TracBrowser for help on using the repository browser.