source: trunk/DataCheck/Tools/get_data.sh@ 19567

Last change on this file since 19567 was 19562, checked in by Daniela Dorner, 5 years ago
updated values
  • Property svn:executable set to *
File size: 31.4 KB
Line 
1#!/bin/bash
2
3# ---------------------------------------------------------------- #
4# README README README README README README README README README #
5# ---------------------------------------------------------------- #
6# #
7# To use this script, you need #
8# - a computer with access to the FACT database in La Palma #
9# - a file with the password of a valid mysql-user #
10# - to define the setup below for #
11# a) the DB access #
12# b) the data you want to have #
13# #
14# To define the setup, search for SETUP in this script and #
15# read the details there #
16# #
17# Per data request, you get up to 3 files: #
18# *_internal.dat #
19# *_collaborators.dat #
20# *_external.dat (only if binning is 20min or nightly) #
21# #
22# Please have in mind that this started as a tool for myself, then #
23# others started using it. Also the script is not yet finalized. #
24# In case you find problems and/or have a feature request, please #
25# send and email to dorner@astro.uni-wuerzburg.de #
26# #
27# ---------------------------------------------------------------- #
28# README README README README README README README README README #
29# ---------------------------------------------------------------- #
30
31
32
33# ToDo (notes DD):
34# ----------------
35# - do conversion to fluxes using FACT Crab spectrum
36# - add < 20121212 data for QLA
37# - new version of zd/th-correction and CU from Aachen when ready
38# nice-to-have
39# - add E2dNdE
40# - functionality to determine start time for seaon-binning
41# - file as attachment to email for download.php
42
43#
44# content of files (wish list):
45# -----------------------------
46# REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
47#
48# internal
49# --------
50# time: time, delta time, start, stop, ontime
51# flux: excrate, excerr, corrate, corerr, CU CUerr, flux, fluxerr,
52# other info on flux: signif, cu-factor, num exc, num sig, num bg
53# other info: zd th R750cor R750ref
54#
55# external (allow only 20min and nightly binning)
56# --------
57# time: time, delta time, start, stop
58# flux: excrate, excerr
59#
60# collaborators
61# -------------
62# time: time, delta time, start, stop, ontime
63# flux: excrate, excerr, corrate, corerr, flux, flux-err, significance
64#
65# additional information to put:
66# ------------------------------
67# timestamp of creation
68# query (for debugging / answering questions)
69# policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
70#
71
72
73function print_policy()
74{
75 echo "#"
76 echo "# Data Usage Policy: "
77 if [ "$expert" == "no" ]
78 then
79 echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
80 echo "# paper (H. Anderhub et al. JINST 8 P6008) and the quick look analysis website "
81 echo "# (https://fact-project.org/monitoring)."
82 echo "#"
83 echo "# References: "
84 echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
85 echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
86 echo "# FACT Performance Paper: http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
87 echo "# http://iopscience.iop.org/1748-0221/9/10/P10012"
88 echo "# FACT quick look analysis: https://fact-project.org/monitoring"
89 echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
90 echo "# If you intend to use data or information from this website, please let us know for reference."
91 else
92 echo "# As a member, associated member or collaborator of the FACT Collaboration, you have access to internal information."
93 echo "# Any publication using FACT internal information has to have the full FACT author list."
94 fi
95 echo "#"
96}
97
98print_selection()
99{
100 echo "#"
101 echo "# Your Selection: "
102 sourcename=`mysql --defaults-file=$sqlpw -s -e "SELECT fSourceName FROM Source WHERE fSourceKey="$source`
103 echo "# Source: "$sourcename
104 echo "# Time range: "$nightmin"-"$nightmax
105 echo "# Time format: "$timeunit
106 if [ $bin -lt 0 ]
107 then
108 unit="night(s)"
109 else
110 if [ $bin -eq 0 ]
111 then
112 unit="periods"
113 else
114 unit="minutes"
115 fi
116 fi
117 binning=`echo $bin | sed -e 's/-//'`" "$unit
118 echo "# Binning: "$binning
119 if [ "$expert" != "no" ]
120 then
121 echo "# Additional Internal Selection: "
122 echo "# "`echo $table | sed -e 's/AnalysisResultsRun//'`"-Analysis was used."
123 if [ "$zdmax" != "" ]
124 then
125 echo "# Maximum Zenith Distance: "$zdmax" degree"
126 fi
127 if [ "$thmax" != "" ]
128 then
129 echo "# Maximum Trigger Threshold: "$thmax" DAC counts"
130 fi
131 if [ "$light" != "" ]
132 then
133 echo "# Light Condition Cut: "$lightcut
134 fi
135 if [ "$dust" != "" ]
136 then
137 echo "# Calima Cut: dust < "$dust" ug/cm3"
138 fi
139 if [ "$usedch" == "yes" ]
140 then
141 echo "# Data quality selection was applied. "
142 if [ "$dch" == "" ]
143 then
144 echo "# Standard data check based on cosmic-ray rate: "$dchstd
145 else
146 echo "# You selected a custom datacheck: "$dch
147 fi
148 fi
149 fi
150 echo "#"
151}
152
153function get_results()
154{
155 # some query parts
156
157 # these values use the Crab spectrum from HESS 2006
158 # hess 2006 http://www.aanda.org/articles/aa/pdf/2006/39/aa5351-06.pdf
159 # PL.SetParameter(1,-2.63);
160 # PL.SetParameter(0, 3.45e-11);
161 case "$table" in
162 "AnalysisResultsRunISDC")
163 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.2)"
164 thfactor="(1.18-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00062)"
165 # ETh: 1070 GeV
166 crabflux="1.89"
167 ;;
168 "AnalysisResultsRunCutsLC")
169 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.5)"
170 thfactor="(1.37-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00118)"
171 # ETh: 785 GeV
172 crabflux="3.14"
173 ;;
174 "AnalysisResultsRunLP")
175 zdfactor="(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))"
176 #zdfactor="(1/(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5)))"
177 thfactor="(1-0.00124/1.21*(fThresholdMinSet-500)*(fThresholdMinSet>=500))"
178 # ETh: 1100 GeV
179 crabflux="1.81"
180 ;;
181 "*")
182 zdfactor="0"
183 thfactor="0"
184 crabflux="0"
185 ;;
186 esac
187
188 # conversion from crab units to fluxes (if a different value is given in setup
189 if [ "$crabfluxconv" != "" ]
190 then
191 crabflux=$crabfluxconv #e-11
192 fi
193 #echo "crabflux: "$crabflux
194 fluxprec=2
195
196 # some names and definitions needed several times below
197 # ontime
198 ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
199 ontime2=" fOnTimeAfterCuts "
200 ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
201 # zd and threshold
202 zenith="fZenithDistance"
203 thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
204 # correction for zd- and th-dependence
205 correvts=" fNumExcEvts/"$zdfactor"/"$thfactor
206 # conversion to CU - stored in each result-table (determined by DD Feb 2019)
207 cufactor="fCU"
208 # some calculations
209 excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
210 CU="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
211 CUerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
212 excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
213 CU2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
214 CUerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
215
216 # columns to be selected
217 # for night-binning
218 ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
219 excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
220 significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
221 numexc="Sum(fNumExcEvts) AS numexc"
222 numsig="Sum(fNumSigEvts) AS numsig"
223 numbg="Sum(fNumBgEvts) AS numbg"
224 excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
225 correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
226 correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
227 cu=" ROUND("$CU", 2) AS cu"
228 cuerr=" ROUND("$CUerr", 2) AS cuerr"
229 flux="ROUND("$CU" * "$crabflux", 2) AS flux"
230 fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
231 # for minute binning
232 ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
233 excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
234 significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
235 numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
236 numsig2="Sum(o.sigevts) AS numsig"
237 numbg2="Sum(o.bgevts) AS numbg"
238 excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
239 correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
240 correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
241 cu2=" ROUND("$CU2", 2) AS cu"
242 cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
243 flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
244 fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
245
246 case $timeunit in
247 mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
248 start=" Mjd(MIN(fRunStart)) AS start"
249 stop=" Mjd(MAX(fRunStop)) AS stop"
250 deltat=$delta" AS deltat"
251 time=" Mjd(MIN(fRunStart))+"$delta" AS time"
252 delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
253 start2=" Mjd(MIN(o.start)) AS start"
254 stop2=" Mjd(MAX(o.stop)) AS stop"
255 deltat2=$delta2" AS deltat"
256 time2=" Mjd(MIN(o.start))+"$delta2" AS time"
257 ;;
258 unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
259 start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
260 stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
261 deltat=$delta" AS deltat"
262 time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
263 delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
264 start2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
265 stop2=" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
266 deltat2=$delta2" AS deltat"
267 time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
268 ;;
269 *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
270 start=" MIN(fRunStart) AS start"
271 stop=" MAX(fRunStop) AS stop"
272 deltat=$delta" AS deltat"
273 time=" addtime(MIN(fRunStart), "$delta") AS time"
274 delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
275 start2=" MIN(o.start) AS start"
276 stop2=" MAX(o.stop) AS stop"
277 deltat2=$delta2" AS deltat"
278 time2=" addtime(MIN(o.start), "$delta2") AS time"
279 ;;
280 esac
281
282 # from and join of query
283 from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
284
285 # data check based on artificial trigger rate
286 # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
287 #dch=" AND fR750Cor/fR750Ref >0.93 "
288 dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
289
290 # put together where-clause of query
291 # time range and source
292 where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
293 where=$where" AND NOT ISNULL(fNumExcEvts) "
294 # some sanity checks
295 where=$where" AND fRunTypeKey=1 "
296 # zd cut
297 if [ "$zdmax" != "" ]
298 then
299 where=$where" AND fZenithDistanceMax < "$zdmax
300 fi
301 # th cut
302 if [ "$thmax" != "" ]
303 then
304 where=$where" AND "$thresh" < "$thmax
305 fi
306 # dust cut
307 if [ "$dust" != "" ]
308 then
309 where=$where" AND fTNGDust<"$dust
310 fi
311 # light condition cut
312 if [ "$light" == "nomoon" ]
313 then
314 lightcut=" fZenithDistanceMoon>90"
315 fi
316 if [ "$light" == "dark" ]
317 then
318 lightcut=" fMoonZenithDistance>90 AND fSunZenithDistance>108 "
319 fi
320 if [ "$light" != "" ]
321 then
322 where=$where" AND "$lightcut
323 fi
324 querybase=$from$where
325
326 if [ "$usedch" == "yes" ]
327 then
328 if [ "$dch" == "" ]
329 then
330 querydch=$dchstd
331 else
332 echo "you are using for datacheck: "$dch
333 querydch=$dch
334 fi
335 else
336 querydch=
337 fi
338
339
340 if [ $bin -le 0 ]
341 then
342 # first part of the query
343 querystart="SELECT "
344 querystart=$querystart" "$time", "$start", "$stop", "
345 # final part of the query
346 if [ $bin -eq 0 ]
347 then
348 orderby=" fPeriod "
349 #querystart=$querystart" fPeriod AS num, "
350 queryend=" GROUP BY fPeriod "
351 else
352 num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
353 orderby=$num
354 #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
355 queryend=" GROUP BY "$num
356 fi
357 #queryend=" GROUP BY num "
358 if [ "$ontimelimit" = "" ]
359 then
360 queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
361 else
362 queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
363 fi
364
365 # internal
366 queryint=$querystart
367 queryint=$queryint" "$excrate", "$correxcrate", "$cu", "$flux", "
368 queryint=$queryint" "$deltat", "$ontime", "
369 queryint=$queryint" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
370 queryint=$queryint" "$significance", "
371 queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
372 queryint=$queryint" "$numexc", "$numsig", "$numbg", "
373 queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
374 queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
375 queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
376 queryint=$queryint" "$querybase" "$querydch" "$queryend
377
378 # for collaborators
379 querycol=$querystart
380 querycol=$querycol" "$excrate", "$correxcrate", "$cu", "$flux", "
381 querycol=$querycol" "$deltat", "$ontime", "
382 querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
383 querycol=$querycol" "$significance
384 querycol=$querycol" "$querybase" "$querydch" "$queryend
385
386 # external
387 # no datacheck applied for external files
388 queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
389
390 else
391 # first part of the query
392 querystart="SELECT "
393 querystart=$querystart" "$time2", "$start2", "$stop2", "
394
395 # final part of the query
396 querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
397 querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
398 querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
399 querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
400 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
401 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
402 querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
403 querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
404
405 # internal
406 queryint=$querystart
407 queryint=$queryint" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
408 queryint=$queryint" "$deltat2", "$ontime2", "
409 queryint=$queryint" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
410 queryint=$queryint" "$significance2", "
411 queryint=$queryint" avg(o.night) AS night, "
412 queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
413 queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
414 queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
415 queryint=$queryint" "$querybase
416
417 # for collaborators
418 querycol=$querystart
419 querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
420 querycol=$querycol" "$deltat2", "$ontime2", "
421 querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
422 querycol=$querycol" "$significance2
423 querycol=$querycol" "$querybase
424
425 # external
426 queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
427
428 fi
429
430 # write file for externals only for allowed binnings
431 if [ $bin -eq 20 ] || [ $bin -eq -1 ]
432 then
433 fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
434 if [ "$overwrite" = "yes" ]
435 then
436 if [ "$mode" != "auto" ]
437 then
438 echo "creating "$fileext" ..."
439 fi
440 echo "# This file was created at "`date` > $fileext
441 print_policy >> $fileext
442 fi
443 print_selection >> $fileext
444 headerext="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] (stop-start)/2["$timeunit"] excess-rate_error[evts/h] "
445 echo $headerext >> $fileext
446 #echo "$queryext"
447 mysql --defaults-file=$sqlpw -s -e "$queryext" >> $fileext
448 #mysql --defaults-file=$sqlpw -e "$queryext"
449 fi
450 if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
451 then
452 return
453 fi
454
455 fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
456 if [ "$overwrite" = "yes" ]
457 then
458 if [ "$mode" != "auto" ]
459 then
460 echo "creating "$fileint" ..."
461 fi
462 echo "# This file was created at "`date` > $fileint
463 print_policy >> $fileint
464 fi
465 print_selection >> $fileint
466 echo "# The following query was used: " >> $fileint
467 echo "# $queryint" >> $fileint
468 echo "#" >> $fileint
469 headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
470 headerint=$headerint" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
471 headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
472 echo $headerint >> $fileint
473 #echo "$queryint"
474 mysql --defaults-file=$sqlpw -s -e "$queryint" >> $fileint
475 #mysql --defaults-file=$sqlpw -e "$queryint"
476 if [ "$mode" == "auto" ]
477 then
478 return
479 fi
480
481 filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
482 if [ "$overwrite" = "yes" ]
483 then
484 echo "creating "$filecol" ..."
485 echo "# This file was created at "`date` > $filecol
486 print_policy >> $filecol
487 fi
488 print_selection >> $filecol
489 #echo "# The following query was used: " >> $filecol
490 #echo "# $querycol" >> $filecol
491 #echo "#" >> $filecol
492 headercol="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
493 headercol=$headercol" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance "
494 echo $headercol >> $filecol
495 #echo "$querycol"
496 mysql --defaults-file=$sqlpw -s -e "$querycol" >> $filecol
497 #mysql --defaults-file=$sqlpw -e "$querycol
498}
499
500# evaluation of command line options (for usage with download.php)
501
502if [ ${#@} -eq 13 ]
503then
504 #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
505 mode="auto"
506 overwrite="yes"
507 # setup for usage with download.php
508 datapath="/home/factwww/dch/data"
509 #datapath="./data"
510 sqlpw=/home/fact/.mysql.pw
511 #sqlpw=/home/fact/.mysql.pw2
512 #host=10.0.100.21
513 #dbname=factdata
514 nightmin=$1
515 nightmax=$2
516 source=$3
517 bin=$4
518 if [ "$bin" == "00" ]
519 then
520 bin=0
521 fi
522 email=$5
523 table=$6
524 timeunit=$7
525 expert=$8
526 usedch=$9 # novalue gives same result as no
527 if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
528 then
529 zdmax=${10}
530 fi
531 if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
532 then
533 thmax=${11}
534 fi
535 if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
536 then
537 light=${12}
538 fi
539 if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
540 then
541 dust=${13}
542 fi
543 name=`echo $email | sed -e 's/@/-at-/'`
544 get_results
545
546 # sending email
547 if [ "$expert" == "yes" ]
548 then
549 cat $fileint | mail -s 'FACT internal data download' -b qla@fact-project.org -r qla@fact-project.org $email
550 else
551 cat $fileext | mail -s 'FACT data download' -b qla@fact-project.org -r qla@fact-project.org $email
552 fi
553
554 exit
555fi
556
557
558
559# -------------------------------------------------------------------------------------- #
560# SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
561# -------------------------------------------------------------------------------------- #
562# #
563# The lines below define the basic setup for the database and give examples and #
564# explanations for the various options available. #
565# The request of the data itself is done with a smaller setup further down. #
566# #
567# -------------------------------------------------------------------------------------- #
568#
569# ----------
570# DB SETUP
571# ----------
572# path to file with mysql setup (user, password, host, db-name)
573sqlpw=/home/$USER/.mysql.pw.local
574sqlpw=/home/$USER/.mysql.pw.ethz.fact
575#
576# -------------
577# BASIC SETUP
578# -------------
579# output path
580path=`dirname $0`
581datapath=$path"/data"
582# create directory for data files
583if ! [ -e $datapath ]
584then
585 mkdir $datapath
586fi
587# time unit
588#timeunit=timestamp # default
589#timeunit=unix
590timeunit=mjd
591# time binning
592# positive values: minutes
593# negative values: days
594# special case 0: period
595# for season binning choose -365 and according start date
596#bin=20 # minutes
597#bin=0 # period
598bin=-1 # nightly
599#bin=-365 # yearly
600# choose analysis
601#table="AnalysisResultsAllQLA" # N/A
602#table="AnalysisResultsRunLP" # QLA
603#table="AnalysisResultsRunISDC" # ISDC
604table="AnalysisResultsRunCutsLC" # CutsLC
605# time range
606nightmin=20111115
607nightmax=20201231
608# overwrite dataset file?
609# (useful to combine different binnings in one file -> set to "no")
610overwrite="yes"
611# optional: require minimal ontime per bin (default 20 min)
612#ontimelimit=30 # 30 min
613ontimelimit= # default 20 min
614# data quality selection
615# if you explicitely don't want a datacheck, you can comment the following line
616usedch="yes"
617# use your own datacheck instead
618# use a line like the following defining your own data quality selection cut
619#dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
620# apply additional predefined cuts
621# light conditions
622#light="nomoon" # only data with no moon (but twilight allowed)
623#light="dark" # only dark night data
624# TNG dust - cut away data with calima
625#dust=1
626#dust=10
627# use different conversion from CU to fluxes
628#crabfluxconv="2.5"
629
630
631# -------------------------------------------------------------------------------------- #
632# SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
633# -------------------------------------------------------------------------------------- #
634# #
635# Adapt the lines below to your needs. #
636# Overwrite default settings above. #
637# The data-request is sent with the line 'get_results'. #
638# Minumum setup: Define source key and name for file. #
639# The list of source keys can be found at #
640# https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
641# More examples can be found further down. #
642# #
643# -------------------------------------------------------------------------------------- #
644
645
646# LC for INTEGRAL Proposal
647table="AnalysisResultsRunCutsLC" # CutsLC
648bin=-1
649# Mrk 421
650source=1
651name="Mrk421_all_nightly"
652get_results
653# Mrk 501
654source=2
655name="Mrk501_all_nightly"
656get_results
657# 1959
658source=7
659name="1959_all_nightly"
660get_results
661# for other INTEGRAL proposal
662# Mrk 421
663bin=0
664source=1
665name="Mrk421_all_period"
666get_results
667
668exit
669
670
671# Crab
672source=5
673nightmin=20121214
674nightmax=20180418
675table="AnalysisResultsRunCutsLC" # CutsLC
676bin=-1
677zdmax=30
678thmax=330
679name="testCrabnightlycheck"
680get_results
681
682exit
683
684# Crab
685source=5
686nightmin=20121214
687nightmax=20180418
688table="AnalysisResultsRunCutsLC" # CutsLC
689bin=20
690name="testCrab20min"
691get_results
692
693exit
694
695# Crab
696source=5
697nightmin=20121214
698nightmax=20180418
699table="AnalysisResultsRunCutsLC" # CutsLC
700bin=-1
701name="testCrabnightly"
702get_results
703
704exit
705
706# Crab
707source=5
708#nightmin=20121214
709#nightmax=20180418
710table="AnalysisResultsRunCutsLC" # CutsLC
711bin=-1
712name="testCraball"
713get_results
714
715exit
716
717
718# Crab
719source=5
720nightmin=20121214
721nightmax=20180418
722table="AnalysisResultsRunCutsLC" # CutsLC
723bin=60
724name="testCrab"
725get_results
726
727exit
728
729
730# Mrk 501
731source=2
732zdmax=90
733thmax=1500
734# new analysis
735table="AnalysisResultsRunCutsLC" # CutsLC
736nightmin=20140622
737nightmax=20140624
738name="Mrk501_5min_Flare_newAnalysis_forHESS"
739bin=5
740get_results
741
742exit
743
744# for 2344 paper with MAGIC
745
746source=3
747zdmax=90
748thmax=1500
749
750# new analysis
751table="AnalysisResultsRunCutsLC" # CutsLC
752nightmin=20160618
753nightmax=20160815
754name="2344_2016flare_newAnalysis"
755bin=-7
756get_results
757overwrite="no"
758bin=-28
759nightmin=20160816
760nightmax=20161031
761get_results
762overwrite="yes"
763
764# ISDC analysis
765table="AnalysisResultsRunISDC"
766nightmin=20160618
767nightmax=20160815
768name="2344_2016flare_stdAnalysis"
769bin=-7
770get_results
771overwrite="no"
772bin=-28
773nightmin=20160816
774nightmax=20161031
775get_results
776overwrite="yes"
777
778
779usedch="no"
780# new analysis
781table="AnalysisResultsRunCutsLC" # CutsLC
782nightmin=20160618
783nightmax=20160815
784name="2344_2016flare_newAnalysis_noDCh"
785bin=-7
786get_results
787overwrite="no"
788bin=-28
789nightmin=20160816
790nightmax=20161031
791get_results
792overwrite="yes"
793
794# ISDC analysis
795table="AnalysisResultsRunISDC"
796nightmin=20160618
797nightmax=20160815
798name="2344_2016flare_stdAnalysis_noDCh"
799bin=-7
800get_results
801overwrite="no"
802bin=-28
803nightmin=20160816
804nightmax=20161031
805get_results
806overwrite="yes"
807
808usedch="yes"
809
810
811exit
812
813# Mrk 501
814source=2
815zdmax=90
816thmax=1500
817# new analysis
818table="AnalysisResultsRunCutsLC" # CutsLC
819nightmin=20140520
820nightmax=20140930
821name="Mrk501_nightly_newAnalysis_forHESS"
822bin=-1
823get_results
824name="Mrk501_7d_newAnalysis_forHESS"
825bin=-7
826get_results
827nightmin=20140623
828nightmax=20140623
829name="Mrk501_5min_FlareNight_newAnalysis_forHESS"
830bin=5
831get_results
832# isdc analysis
833thmax=850
834table="AnalysisResultsRunISDC"
835nightmin=20140520
836nightmax=20140930
837name="Mrk501_nightly_stdAnalysis_forHESS"
838bin=-1
839get_results
840name="Mrk501_7d_stdAnalysis_forHESS"
841bin=-7
842get_results
843nightmin=20140623
844nightmax=20140623
845name="Mrk501_5min_FlareNight_stdAnalysis_forHESS"
846bin=5
847get_results
848# qla
849thmax=500
850table="AnalysisResultsRunLP"
851nightmin=20140520
852nightmax=20140930
853name="Mrk501_nightly_QLA_forHESS"
854bin=-1
855get_results
856
857exit
858
859# 501 MAGIC
860source=2
861name="Mrk501_2014_forMAGIC"
862bin=-1
863crabfluxconv="2.87" # using threshold of 830GeV
864crabfluxconv="3.01" # using threshold of 830GeV and MAGIC Crab spectrum
865nightmin=20140714
866nightmax=20140805
867get_results
868
869name="Mrk501_2014_forMAGIC_20min"
870bin=20
871get_results
872
873
874exit
875
876bin=30
877name="Mrk501_2014_forMAGIC30"
878get_results
879
880bin=0
881name="P"
882nightmin=20140501
883nightmax=20140930
884get_results
885
886bin=20
887nightmin=20140623
888nightmax=20140623
889name="Mrk501_test"
890get_results
891
892
893# end script here
894exit
895
896
897
898#
899# more examples
900#
901
902# Mrk 421
903source=1
904name="Mrk421_nightly"
905bin=-1
906get_results
907name="Mrk421_20min"
908bin=20
909get_results
910name="Mrk421_3d"
911bin=-3
912get_results
913name="Mrk421_10d"
914bin=-10
915get_results
916name="Mrk421_period"
917bin=0
918get_results
919
920
921
922# Mrk 501
923source=2
924name="Mrk501_nightly"
925bin=-1
926get_results
927name="Mrk501_20min"
928bin=20
929get_results
930name="Mrk501_3d"
931bin=-3
932get_results
933name="Mrk501_10d"
934bin=-10
935get_results
936name="Mrk501_period"
937bin=0
938get_results
939
940
941
942# 2344
943source=3
944name="2344_nightly"
945bin=-1
946get_results
947name="2344_20min"
948bin=20
949get_results
950name="2344_period"
951bin=0
952get_results
953
954
955
956# 1959
957source=7
958name="1959_nightly"
959bin=-1
960get_results
961name="1959_20min"
962bin=20
963get_results
964name="1959_period"
965bin=0
966get_results
967
968
969
970# 0323
971source=12
972name="0323_nightly"
973bin=-1
974get_results
975name="0323_20min"
976bin=20
977get_results
978name="0323_period"
979bin=0
980get_results
981
982
983
984# crab
985source=5
986name="Crab_nightly"
987bin=-1
988get_results
989name="Crab_20min"
990bin=20
991get_results
992name="Crab_period"
993bin=0
994get_results
995name="Crab_season"
996bin=-365
997nightmin=20110716
998nightmax=20180716
999get_results
1000
1001
1002
1003name="1959_2016"
1004source=7
1005bin=-1
1006nightmin=20160201
1007nightmax=20161105
1008get_results
1009
1010name="1959_all_variable"
1011overwrite="no"
1012source=7
1013bin=-365
1014nightmin=20120201
1015nightmax=20130131
1016get_results
1017nightmin=20130201
1018nightmax=20140131
1019get_results
1020nightmin=20140201
1021nightmax=20150131
1022get_results
1023bin=0
1024nightmin=20150201
1025nightmax=20160131
1026get_results
1027bin=-1
1028nightmin=20160201
1029nightmax=20170131
1030get_results
1031bin=0
1032nightmin=20170201
1033nightmax=20180131
1034get_results
1035
1036
1037
1038overwrite="yes"
1039name="1959_all_variable2"
1040overwrite="no"
1041source=7
1042bin=-365
1043nightmin=20120201
1044nightmax=20130131
1045get_results
1046nightmin=20130201
1047nightmax=20140131
1048get_results
1049nightmin=20140201
1050nightmax=20150131
1051get_results
1052bin=0
1053nightmin=20150201
1054nightmax=20160131
1055get_results
1056bin=-1
1057nightmin=20160201
1058nightmax=20160817
1059get_results
1060bin=0
1061nightmin=20160818
1062nightmax=20180131
1063get_results
1064
1065
1066
1067overwrite="yes"
1068bin=0
1069source=3
1070name="2344period"
1071get_results
1072
1073
1074
1075# flare night (HESS)
1076name="Mrk501_10min_flarenight"
1077source=2
1078bin=10
1079nightmin=20140623
1080nightmax=20140623
1081get_results
1082
1083
1084
1085# flare night (HESS)
1086name="Mrk501_5min_flarenight"
1087source=2
1088bin=5
1089nightmin=20140623
1090nightmax=20140623
1091get_results
1092
1093
1094
1095
1096# full sample
1097name="Mrk421_all_nightly"
1098source=1
1099get_results
1100
1101name="Mrk501_all_nightly"
1102source=2
1103get_results
1104
1105name="1959_all_nightly"
1106source=7
1107get_results
1108
1109name="2344_all_nightly"
1110source=3
1111get_results
1112
1113
1114
1115name="HESE20160427"
1116source=19
1117nightmin=20160425
1118bin=-10
1119get_results
1120
1121name="AMON20160731"
1122source=21
1123nightmin=20160730
1124bin=-10
1125get_results
1126
1127
1128
Note: See TracBrowser for help on using the repository browser.