source: trunk/DataCheck/Tools/get_data.sh@ 19521

Last change on this file since 19521 was 19448, checked in by Daniela Dorner, 6 years ago
fixed issue with datacheck
  • Property svn:executable set to *
File size: 29.9 KB
Line 
1#!/bin/bash
2
3# ---------------------------------------------------------------- #
4# README README README README README README README README README #
5# ---------------------------------------------------------------- #
6# #
7# To use this script, you need #
8# - a computer with access to the FACT database in La Palma #
9# - a file with the password of a valid mysql-user #
10# - to define the setup below for #
11# a) the DB access #
12# b) the data you want to have #
13# #
14# To define the setup, search for SETUP in this script and #
15# read the details there #
16# #
17# Per data request, you get up to 3 files: #
18# *_internal.dat #
19# *_collaborators.dat #
20# *_external.dat (only if binning is 20min or nightly) #
21# #
22# Please have in mind that this started as a tool for myself, then #
23# others started using it. Also the script is not yet finalized. #
24# In case you find problems and/or have a feature request, please #
25# send and email to dorner@astro.uni-wuerzburg.de #
26# #
27# ---------------------------------------------------------------- #
28# README README README README README README README README README #
29# ---------------------------------------------------------------- #
30
31
32
33# ToDo (notes DD):
34# ----------------
35# - do conversion to fluxes using FACT Crab spectrum
36# - add < 20121212 data for QLA
37# - new version of zd/th-correction and CU from Aachen when ready
38# nice-to-have
39# - add E2dNdE
40# - functionality to determine start time for seaon-binning
41# - file as attachment to email for download.php
42
43#
44# content of files (wish list):
45# -----------------------------
46# REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
47#
48# internal
49# --------
50# time: time, delta time, start, stop, ontime
51# flux: excrate, excerr, corrate, corerr, CU CUerr, flux, fluxerr,
52# other info on flux: signif, cu-factor, num exc, num sig, num bg
53# other info: zd th R750cor R750ref
54#
55# external (allow only 20min and nightly binning)
56# --------
57# time: time, delta time, start, stop
58# flux: excrate, excerr
59#
60# collaborators
61# -------------
62# time: time, delta time, start, stop, ontime
63# flux: excrate, excerr, corrate, corerr, flux, flux-err, significance
64#
65# additional information to put:
66# ------------------------------
67# timestamp of creation
68# query (for debugging / answering questions)
69# policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
70#
71
72
73function print_policy()
74{
75 echo "#"
76 echo "# Data Usage Policy: "
77 if [ "$expert" == "no" ]
78 then
79 echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
80 echo "# paper (H. Anderhub et al. JINST 8 P6008) and the quick look analysis website "
81 echo "# (https://fact-project.org/monitoring)."
82 echo "#"
83 echo "# References: "
84 echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
85 echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
86 echo "# FACT Performance Paper: http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
87 echo "# http://iopscience.iop.org/1748-0221/9/10/P10012"
88 echo "# FACT quick look analysis: https://fact-project.org/monitoring"
89 echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
90 echo "# If you intend to use data or information from this website, please let us know for reference."
91 else
92 echo "# As a member, associated member or collaborator of the FACT Collaboration, you have access to internal information."
93 echo "# Any publication using FACT internal information has to have the full FACT author list."
94 fi
95 echo "#"
96}
97
98print_selection()
99{
100 echo "#"
101 echo "# Your Selection: "
102 sourcename=`mysql --defaults-file=$sqlpw -s -e "SELECT fSourceName FROM Source WHERE fSourceKey="$source`
103 echo "# Source: "$sourcename
104 echo "# Time range: "$nightmin"-"$nightmax
105 echo "# Time format: "$timeunit
106 if [ $bin -lt 0 ]
107 then
108 unit="night(s)"
109 else
110 if [ $bin -eq 0 ]
111 then
112 unit="periods"
113 else
114 unit="minutes"
115 fi
116 fi
117 binning=`echo $bin | sed -e 's/-//'`" "$unit
118 echo "# Binning: "$binning
119 if [ "$expert" != "no" ]
120 then
121 echo "# Additional Internal Selection: "
122 echo "# "`echo $table | sed -e 's/AnalysisResultsRun//'`"-Analysis was used."
123 if [ "$zdmax" != "" ]
124 then
125 echo "# Maximum Zenith Distance: "$zdmax" degree"
126 fi
127 if [ "$thmax" != "" ]
128 then
129 echo "# Maximum Trigger Threshold: "$thmax" DAC counts"
130 fi
131 if [ "$light" != "" ]
132 then
133 echo "# Light Condition Cut: "$lightcut
134 fi
135 if [ "$dust" != "" ]
136 then
137 echo "# Calima Cut: dust < "$dust" ug/cm3"
138 fi
139 if [ "$usedch" == "yes" ]
140 then
141 echo "# Data quality selection was applied. "
142 if [ "$dch" == "" ]
143 then
144 echo "# Standard data check based on cosmic-ray rate: "$dchstd
145 else
146 echo "# You selected a custom datacheck: "$dch
147 fi
148 fi
149 fi
150 echo "#"
151}
152
153function get_results()
154{
155 # some query parts
156
157 # these values use the Crab spectrum from HESS 2006
158 case "$table" in
159 "AnalysisResultsRunISDC")
160 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.2)"
161 thfactor="(1.18-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00062)"
162 # ETh: 764 GeV
163 crabflux="3.3"
164 ;;
165 "AnalysisResultsRunCutsLC")
166 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.5)"
167 thfactor="(1.37-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00118)"
168 # ETh: 543 GeV
169 crabflux="5.7"
170 ;;
171 "AnalysisResultsRunLP")
172 zdfactor="(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))"
173 #zdfactor="(1/(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5)))"
174 thfactor="(1-0.00124/1.21*(fThresholdMinSet-500)*(fThresholdMinSet>=500))"
175 # ETh: 715 GeV
176 crabflux="3.6"
177 ;;
178 "*")
179 zdfactor="0"
180 thfactor="0"
181 crabflux="0"
182 ;;
183 esac
184
185 # conversion from crab units to fluxes (if a different value is given in setup
186 if [ "$crabfluxconv" != "" ]
187 then
188 crabflux=$crabfluxconv #e-11
189 fi
190 #echo "crabflux: "$crabflux
191 fluxprec=2
192
193 # some names and definitions needed several times below
194 # ontime
195 ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
196 ontime2=" fOnTimeAfterCuts "
197 ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
198 # zd and threshold
199 zenith="fZenithDistance"
200 thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
201 # correction for zd- and th-dependence
202 correvts=" fNumExcEvts/"$zdfactor"/"$thfactor
203 # conversion to CU - stored in each result-table (determined by DD Feb 2019)
204 cufactor="fCU"
205 # some calculations
206 excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
207 CU="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
208 CUerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
209 excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
210 CU2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
211 CUerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
212
213 # columns to be selected
214 # for night-binning
215 ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
216 excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
217 significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
218 numexc="Sum(fNumExcEvts) AS numexc"
219 numsig="Sum(fNumSigEvts) AS numsig"
220 numbg="Sum(fNumBgEvts) AS numbg"
221 excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
222 correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
223 correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
224 cu=" ROUND("$CU", 2) AS cu"
225 cuerr=" ROUND("$CUerr", 2) AS cuerr"
226 flux="ROUND("$CU" * "$crabflux", 2) AS flux"
227 fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
228 # for minute binning
229 ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
230 excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
231 significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
232 numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
233 numsig2="Sum(o.sigevts) AS numsig"
234 numbg2="Sum(o.bgevts) AS numbg"
235 excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
236 correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
237 correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
238 cu2=" ROUND("$CU2", 2) AS cu"
239 cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
240 flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
241 fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
242
243 case $timeunit in
244 mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
245 start=" Mjd(MIN(fRunStart)) AS start"
246 stop=" Mjd(MAX(fRunStop)) AS stop"
247 deltat=$delta" AS deltat"
248 time=" Mjd(MIN(fRunStart))+"$delta" AS time"
249 delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
250 start2=" Mjd(MIN(o.start)) AS start"
251 stop2=" Mjd(MAX(o.stop)) AS stop"
252 deltat2=$delta2" AS deltat"
253 time2=" Mjd(MIN(o.start))+"$delta2" AS time"
254 ;;
255 unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
256 start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
257 stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
258 deltat=$delta" AS deltat"
259 time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
260 delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
261 start2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
262 stop2=" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
263 deltat2=$delta2" AS deltat"
264 time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
265 ;;
266 *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
267 start=" MIN(fRunStart) AS start"
268 stop=" MAX(fRunStop) AS stop"
269 deltat=$delta" AS deltat"
270 time=" addtime(MIN(fRunStart), "$delta") AS time"
271 delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
272 start2=" MIN(o.start) AS start"
273 stop2=" MAX(o.stop) AS stop"
274 deltat2=$delta2" AS deltat"
275 time2=" addtime(MIN(o.start), "$delta2") AS time"
276 ;;
277 esac
278
279 # from and join of query
280 from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
281
282 # data check based on artificial trigger rate
283 # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
284 #dch=" AND fR750Cor/fR750Ref >0.93 "
285 dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
286
287 # put together where-clause of query
288 # time range and source
289 where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
290 where=$where" AND NOT ISNULL(fNumExcEvts) "
291 # some sanity checks
292 where=$where" AND fRunTypeKey=1 "
293 # zd cut
294 if [ "$zdmax" != "" ]
295 then
296 where=$where" AND fZenithDistanceMax < "$zdmax
297 fi
298 # th cut
299 if [ "$thmax" != "" ]
300 then
301 where=$where" AND "$thresh" < "$thmax
302 fi
303 # dust cut
304 if [ "$dust" != "" ]
305 then
306 where=$where" AND fTNGDust<"$dust
307 fi
308 # light condition cut
309 if [ "$light" == "nomoon" ]
310 then
311 lightcut=" fZenithDistanceMoon>90"
312 fi
313 if [ "$light" == "dark" ]
314 then
315 lightcut=" fMoonZenithDistance>90 AND fSunZenithDistance>108 "
316 fi
317 if [ "$light" != "" ]
318 then
319 where=$where" AND "$lightcut
320 fi
321 querybase=$from$where
322
323 if [ "$usedch" == "yes" ]
324 then
325 if [ "$dch" == "" ]
326 then
327 querydch=$dchstd
328 else
329 echo "you are using for datacheck: "$dch
330 querydch=$dch
331 fi
332 else
333 querydch=
334 fi
335
336
337 if [ $bin -le 0 ]
338 then
339 # first part of the query
340 querystart="SELECT "
341 querystart=$querystart" "$time", "$start", "$stop", "
342 # final part of the query
343 if [ $bin -eq 0 ]
344 then
345 orderby=" fPeriod "
346 #querystart=$querystart" fPeriod AS num, "
347 queryend=" GROUP BY fPeriod "
348 else
349 num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
350 orderby=$num
351 #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
352 queryend=" GROUP BY "$num
353 fi
354 #queryend=" GROUP BY num "
355 if [ "$ontimelimit" = "" ]
356 then
357 queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
358 else
359 queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
360 fi
361
362 # internal
363 queryint=$querystart
364 queryint=$queryint" "$excrate", "$correxcrate", "$cu", "$flux", "
365 queryint=$queryint" "$deltat", "$ontime", "
366 queryint=$queryint" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
367 queryint=$queryint" "$significance", "
368 queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
369 queryint=$queryint" "$numexc", "$numsig", "$numbg", "
370 queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
371 queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
372 queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
373 queryint=$queryint" "$querybase" "$querydch" "$queryend
374
375 # for collaborators
376 querycol=$querystart
377 querycol=$querycol" "$excrate", "$correxcrate", "$cu", "$flux", "
378 querycol=$querycol" "$deltat", "$ontime", "
379 querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
380 querycol=$querycol" "$significance
381 querycol=$querycol" "$querybase" "$querydch" "$queryend
382
383 # external
384 # no datacheck applied for external files
385 queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
386
387 else
388 # first part of the query
389 querystart="SELECT "
390 querystart=$querystart" "$time2", "$start2", "$stop2", "
391
392 # final part of the query
393 querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
394 querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
395 querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
396 querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
397 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
398 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
399 querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
400 querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
401
402 # internal
403 queryint=$querystart
404 queryint=$queryint" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
405 queryint=$queryint" "$deltat2", "$ontime2", "
406 queryint=$queryint" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
407 queryint=$queryint" "$significance2", "
408 queryint=$queryint" avg(o.night) AS night, "
409 queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
410 queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
411 queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
412 queryint=$queryint" "$querybase
413
414 # for collaborators
415 querycol=$querystart
416 querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
417 querycol=$querycol" "$deltat2", "$ontime2", "
418 querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
419 querycol=$querycol" "$significance2
420 querycol=$querycol" "$querybase
421
422 # external
423 queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
424
425 fi
426
427 # write file for externals only for allowed binnings
428 if [ $bin -eq 20 ] || [ $bin -eq -1 ]
429 then
430 fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
431 if [ "$overwrite" = "yes" ]
432 then
433 if [ "$mode" != "auto" ]
434 then
435 echo "creating "$fileext" ..."
436 fi
437 echo "# This file was created at "`date` > $fileext
438 print_policy >> $fileext
439 fi
440 print_selection >> $fileext
441 headerext="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] (stop-start)/2["$timeunit"] excess-rate_error[evts/h] "
442 echo $headerext >> $fileext
443 #echo "$queryext"
444 mysql --defaults-file=$sqlpw -s -e "$queryext" >> $fileext
445 #mysql --defaults-file=$sqlpw -e "$queryext"
446 fi
447 if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
448 then
449 return
450 fi
451
452 fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
453 if [ "$overwrite" = "yes" ]
454 then
455 if [ "$mode" != "auto" ]
456 then
457 echo "creating "$fileint" ..."
458 fi
459 echo "# This file was created at "`date` > $fileint
460 print_policy >> $fileint
461 fi
462 print_selection >> $fileint
463 echo "# The following query was used: " >> $fileint
464 echo "# $queryint" >> $fileint
465 echo "#" >> $fileint
466 headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
467 headerint=$headerint" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
468 headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
469 echo $headerint >> $fileint
470 #echo "$queryint"
471 mysql --defaults-file=$sqlpw -s -e "$queryint" >> $fileint
472 #mysql --defaults-file=$sqlpw -e "$queryint"
473 if [ "$mode" == "auto" ]
474 then
475 return
476 fi
477
478 filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
479 if [ "$overwrite" = "yes" ]
480 then
481 echo "creating "$filecol" ..."
482 echo "# This file was created at "`date` > $filecol
483 print_policy >> $filecol
484 fi
485 print_selection >> $filecol
486 #echo "# The following query was used: " >> $filecol
487 #echo "# $querycol" >> $filecol
488 #echo "#" >> $filecol
489 headercol="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
490 headercol=$headercol" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance "
491 echo $headercol >> $filecol
492 #echo "$querycol"
493 mysql --defaults-file=$sqlpw -s -e "$querycol" >> $filecol
494 #mysql --defaults-file=$sqlpw -e "$querycol
495}
496
497# evaluation of command line options (for usage with download.php)
498
499if [ ${#@} -eq 13 ]
500then
501 #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
502 mode="auto"
503 overwrite="yes"
504 # setup for usage with download.php
505 datapath="/home/factwww/dch/data"
506 #datapath="./data"
507 sqlpw=/home/fact/.mysql.pw
508 #sqlpw=/home/fact/.mysql.pw2
509 #host=10.0.100.21
510 #dbname=factdata
511 nightmin=$1
512 nightmax=$2
513 source=$3
514 bin=$4
515 if [ "$bin" == "00" ]
516 then
517 bin=0
518 fi
519 email=$5
520 table=$6
521 timeunit=$7
522 expert=$8
523 usedch=$9 # novalue gives same result as no
524 if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
525 then
526 zdmax=${10}
527 fi
528 if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
529 then
530 thmax=${11}
531 fi
532 if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
533 then
534 light=${12}
535 fi
536 if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
537 then
538 dust=${13}
539 fi
540 name=`echo $email | sed -e 's/@/-at-/'`
541 get_results
542
543 # sending email
544 if [ "$expert" == "yes" ]
545 then
546 cat $fileint | mail -s 'FACT internal data download' -b qla@fact-project.org -r qla@fact-project.org $email
547 else
548 cat $fileext | mail -s 'FACT data download' -b qla@fact-project.org -r qla@fact-project.org $email
549 fi
550
551 exit
552fi
553
554
555
556# -------------------------------------------------------------------------------------- #
557# SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
558# -------------------------------------------------------------------------------------- #
559# #
560# The lines below define the basic setup for the database and give examples and #
561# explanations for the various options available. #
562# The request of the data itself is done with a smaller setup further down. #
563# #
564# -------------------------------------------------------------------------------------- #
565#
566# ----------
567# DB SETUP
568# ----------
569# path to file with mysql setup (user, password, host, db-name)
570sqlpw=/home/$USER/.mysql.pw.local
571#
572# -------------
573# BASIC SETUP
574# -------------
575# output path
576path=`dirname $0`
577datapath=$path"/data"
578# create directory for data files
579if ! [ -e $datapath ]
580then
581 mkdir $datapath
582fi
583# time unit
584#timeunit=timestamp # default
585#timeunit=unix
586timeunit=mjd
587# time binning
588# positive values: minutes
589# negative values: days
590# special case 0: period
591# for season binning choose -365 and according start date
592#bin=20 # minutes
593#bin=0 # period
594bin=-1 # nightly
595#bin=-365 # yearly
596# choose analysis
597#table="AnalysisResultsAllQLA" # N/A
598#table="AnalysisResultsRunLP" # QLA
599#table="AnalysisResultsRunISDC" # ISDC
600table="AnalysisResultsRunCutsLC" # CutsLC
601# time range
602nightmin=20111115
603nightmax=20201231
604# overwrite dataset file?
605# (useful to combine different binnings in one file -> set to "no")
606overwrite="yes"
607# optional: require minimal ontime per bin (default 20 min)
608#ontimelimit=30 # 30 min
609ontimelimit= # default 20 min
610# data quality selection
611# if you explicitely don't want a datacheck, you can comment the following line
612usedch="yes"
613# use your own datacheck instead
614# use a line like the following defining your own data quality selection cut
615#dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
616# apply additional predefined cuts
617# light conditions
618#light="nomoon" # only data with no moon (but twilight allowed)
619#light="dark" # only dark night data
620# TNG dust - cut away data with calima
621#dust=1
622#dust=10
623# use different conversion from CU to fluxes
624#crabfluxconv="2.5"
625
626
627# -------------------------------------------------------------------------------------- #
628# SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
629# -------------------------------------------------------------------------------------- #
630# #
631# Adapt the lines below to your needs. #
632# Overwrite default settings above. #
633# The data-request is sent with the line 'get_results'. #
634# Minumum setup: Define source key and name for file. #
635# The list of source keys can be found at #
636# https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
637# More examples can be found further down. #
638# #
639# -------------------------------------------------------------------------------------- #
640
641# for 2344 paper with MAGIC
642
643source=3
644zdmax=90
645thmax=1500
646
647# new analysis
648table="AnalysisResultsRunCutsLC" # CutsLC
649nightmin=20160618
650nightmax=20160815
651name="2344_2016flare_newAnalysis"
652bin=-7
653get_results
654overwrite="no"
655bin=-28
656nightmin=20160816
657nightmax=20161031
658get_results
659overwrite="yes"
660
661# ISDC analysis
662table="AnalysisResultsRunISDC"
663nightmin=20160618
664nightmax=20160815
665name="2344_2016flare_stdAnalysis"
666bin=-7
667get_results
668overwrite="no"
669bin=-28
670nightmin=20160816
671nightmax=20161031
672get_results
673overwrite="yes"
674
675
676usedch="no"
677# new analysis
678table="AnalysisResultsRunCutsLC" # CutsLC
679nightmin=20160618
680nightmax=20160815
681name="2344_2016flare_newAnalysis_noDCh"
682bin=-7
683get_results
684overwrite="no"
685bin=-28
686nightmin=20160816
687nightmax=20161031
688get_results
689overwrite="yes"
690
691# ISDC analysis
692table="AnalysisResultsRunISDC"
693nightmin=20160618
694nightmax=20160815
695name="2344_2016flare_stdAnalysis_noDCh"
696bin=-7
697get_results
698overwrite="no"
699bin=-28
700nightmin=20160816
701nightmax=20161031
702get_results
703overwrite="yes"
704
705usedch="yes"
706
707
708exit
709
710# Mrk 501
711source=2
712zdmax=90
713thmax=1500
714# new analysis
715table="AnalysisResultsRunCutsLC" # CutsLC
716nightmin=20140520
717nightmax=20140930
718name="Mrk501_nightly_newAnalysis_forHESS"
719bin=-1
720get_results
721name="Mrk501_7d_newAnalysis_forHESS"
722bin=-7
723get_results
724nightmin=20140623
725nightmax=20140623
726name="Mrk501_5min_FlareNight_newAnalysis_forHESS"
727bin=5
728get_results
729# isdc analysis
730thmax=850
731table="AnalysisResultsRunISDC"
732nightmin=20140520
733nightmax=20140930
734name="Mrk501_nightly_stdAnalysis_forHESS"
735bin=-1
736get_results
737name="Mrk501_7d_stdAnalysis_forHESS"
738bin=-7
739get_results
740nightmin=20140623
741nightmax=20140623
742name="Mrk501_5min_FlareNight_stdAnalysis_forHESS"
743bin=5
744get_results
745# qla
746thmax=500
747table="AnalysisResultsRunLP"
748nightmin=20140520
749nightmax=20140930
750name="Mrk501_nightly_QLA_forHESS"
751bin=-1
752get_results
753
754exit
755
756# 501 MAGIC
757source=2
758name="Mrk501_2014_forMAGIC"
759bin=-1
760crabfluxconv="2.87" # using threshold of 830GeV
761crabfluxconv="3.01" # using threshold of 830GeV and MAGIC Crab spectrum
762nightmin=20140714
763nightmax=20140805
764get_results
765
766name="Mrk501_2014_forMAGIC_20min"
767bin=20
768get_results
769
770
771exit
772
773bin=30
774name="Mrk501_2014_forMAGIC30"
775get_results
776
777bin=0
778name="P"
779nightmin=20140501
780nightmax=20140930
781get_results
782
783bin=20
784nightmin=20140623
785nightmax=20140623
786name="Mrk501_test"
787get_results
788
789
790# end script here
791exit
792
793
794
795#
796# more examples
797#
798
799# Mrk 421
800source=1
801name="Mrk421_nightly"
802bin=-1
803get_results
804name="Mrk421_20min"
805bin=20
806get_results
807name="Mrk421_3d"
808bin=-3
809get_results
810name="Mrk421_10d"
811bin=-10
812get_results
813name="Mrk421_period"
814bin=0
815get_results
816
817
818
819# Mrk 501
820source=2
821name="Mrk501_nightly"
822bin=-1
823get_results
824name="Mrk501_20min"
825bin=20
826get_results
827name="Mrk501_3d"
828bin=-3
829get_results
830name="Mrk501_10d"
831bin=-10
832get_results
833name="Mrk501_period"
834bin=0
835get_results
836
837
838
839# 2344
840source=3
841name="2344_nightly"
842bin=-1
843get_results
844name="2344_20min"
845bin=20
846get_results
847name="2344_period"
848bin=0
849get_results
850
851
852
853# 1959
854source=7
855name="1959_nightly"
856bin=-1
857get_results
858name="1959_20min"
859bin=20
860get_results
861name="1959_period"
862bin=0
863get_results
864
865
866
867# 0323
868source=12
869name="0323_nightly"
870bin=-1
871get_results
872name="0323_20min"
873bin=20
874get_results
875name="0323_period"
876bin=0
877get_results
878
879
880
881# crab
882source=5
883name="Crab_nightly"
884bin=-1
885get_results
886name="Crab_20min"
887bin=20
888get_results
889name="Crab_period"
890bin=0
891get_results
892name="Crab_season"
893bin=-365
894nightmin=20110716
895nightmax=20180716
896get_results
897
898
899
900name="1959_2016"
901source=7
902bin=-1
903nightmin=20160201
904nightmax=20161105
905get_results
906
907name="1959_all_variable"
908overwrite="no"
909source=7
910bin=-365
911nightmin=20120201
912nightmax=20130131
913get_results
914nightmin=20130201
915nightmax=20140131
916get_results
917nightmin=20140201
918nightmax=20150131
919get_results
920bin=0
921nightmin=20150201
922nightmax=20160131
923get_results
924bin=-1
925nightmin=20160201
926nightmax=20170131
927get_results
928bin=0
929nightmin=20170201
930nightmax=20180131
931get_results
932
933
934
935overwrite="yes"
936name="1959_all_variable2"
937overwrite="no"
938source=7
939bin=-365
940nightmin=20120201
941nightmax=20130131
942get_results
943nightmin=20130201
944nightmax=20140131
945get_results
946nightmin=20140201
947nightmax=20150131
948get_results
949bin=0
950nightmin=20150201
951nightmax=20160131
952get_results
953bin=-1
954nightmin=20160201
955nightmax=20160817
956get_results
957bin=0
958nightmin=20160818
959nightmax=20180131
960get_results
961
962
963
964overwrite="yes"
965bin=0
966source=3
967name="2344period"
968get_results
969
970
971
972# flare night (HESS)
973name="Mrk501_10min_flarenight"
974source=2
975bin=10
976nightmin=20140623
977nightmax=20140623
978get_results
979
980
981
982# flare night (HESS)
983name="Mrk501_5min_flarenight"
984source=2
985bin=5
986nightmin=20140623
987nightmax=20140623
988get_results
989
990
991
992
993# full sample
994name="Mrk421_all_nightly"
995source=1
996get_results
997
998name="Mrk501_all_nightly"
999source=2
1000get_results
1001
1002name="1959_all_nightly"
1003source=7
1004get_results
1005
1006name="2344_all_nightly"
1007source=3
1008get_results
1009
1010
1011
1012name="HESE20160427"
1013source=19
1014nightmin=20160425
1015bin=-10
1016get_results
1017
1018name="AMON20160731"
1019source=21
1020nightmin=20160730
1021bin=-10
1022get_results
1023
1024
1025
Note: See TracBrowser for help on using the repository browser.