source: trunk/DataCheck/Tools/get_data.sh@ 19718

Last change on this file since 19718 was 19570, checked in by Daniela Dorner, 5 years ago
fixed flux units
  • Property svn:executable set to *
File size: 33.1 KB
Line 
1#!/bin/bash
2
3# ---------------------------------------------------------------- #
4# README README README README README README README README README #
5# ---------------------------------------------------------------- #
6# #
7# To use this script, you need #
8# - a computer with access to the FACT database in La Palma #
9# - a file with the password of a valid mysql-user #
10# - to define the setup below for #
11# a) the DB access #
12# b) the data you want to have #
13# #
14# To define the setup, search for SETUP in this script and #
15# read the details there #
16# #
17# Per data request, you get up to 3 files: #
18# *_internal.dat #
19# *_collaborators.dat #
20# *_external.dat (only if binning is 20min or nightly) #
21# #
22# Please have in mind that this started as a tool for myself, then #
23# others started using it. Also the script is not yet finalized. #
24# In case you find problems and/or have a feature request, please #
25# send and email to dorner@astro.uni-wuerzburg.de #
26# #
27# ---------------------------------------------------------------- #
28# README README README README README README README README README #
29# ---------------------------------------------------------------- #
30
31
32
33# ToDo (notes DD):
34# ----------------
35# - do conversion to fluxes using FACT Crab spectrum
36# - add < 20121212 data for QLA
37# - new version of zd/th-correction and CU from Aachen when ready
38# nice-to-have
39# - add E2dNdE
40# - functionality to determine start time for seaon-binning
41# - file as attachment to email for download.php
42
43#
44# content of files (wish list):
45# -----------------------------
46# REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
47#
48# internal
49# --------
50# time: time, delta time, start, stop, ontime
51# flux: excrate, excerr, CU, CUerr, flux, fluxerr corrate, corerr, CUcor CUcorerr, fluxcor, fluxcorerr,
52# other info on flux: signif, cu-factor, num exc, num sig, num bg
53# other info: zd th R750cor R750ref
54#
55# external (allow only 20min and nightly binning)
56# --------
57# time: time, delta time, start, stop
58# flux: excrate, excerr
59#
60# collaborators
61# -------------
62# time: time, delta time, start, stop, ontime
63# flux: excrate, excerr, corrate, corerr, CUcor CUcorerr, fluxcor, fluxcorrerr, significance
64#
65# additional information to put:
66# ------------------------------
67# timestamp of creation
68# query (for debugging / answering questions)
69# policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
70#
71
72
73function print_policy()
74{
75 echo "#"
76 echo "# Data Usage Policy: "
77 if [ "$expert" == "no" ]
78 then
79 echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
80 echo "# paper (H. Anderhub et al. JINST 8 P6008) and the quick look analysis website "
81 echo "# (https://fact-project.org/monitoring)."
82 echo "#"
83 echo "# References: "
84 echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
85 echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
86 echo "# FACT Performance Paper: http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
87 echo "# http://iopscience.iop.org/1748-0221/9/10/P10012"
88 echo "# FACT quick look analysis: https://fact-project.org/monitoring"
89 echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
90 echo "# If you intend to use data or information from this website, please let us know for reference."
91 else
92 echo "# As a member, associated member or collaborator of the FACT Collaboration, you have access to internal information."
93 echo "# Any publication using FACT internal information has to have the full FACT author list."
94 fi
95 echo "#"
96}
97
98print_selection()
99{
100 echo "#"
101 echo "# Your Selection: "
102 sourcename=`mysql --defaults-file=$sqlpw -s -e "SELECT fSourceName FROM Source WHERE fSourceKey="$source`
103 echo "# Source: "$sourcename
104 echo "# Time range: "$nightmin"-"$nightmax
105 echo "# Time format: "$timeunit
106 if [ $bin -lt 0 ]
107 then
108 unit="night(s)"
109 else
110 if [ $bin -eq 0 ]
111 then
112 unit="periods"
113 else
114 unit="minutes"
115 fi
116 fi
117 binning=`echo $bin | sed -e 's/-//'`" "$unit
118 echo "# Binning: "$binning
119 if [ "$expert" != "no" ]
120 then
121 echo "# Additional Internal Selection: "
122 echo "# "`echo $table | sed -e 's/AnalysisResultsRun//'`"-Analysis was used."
123 if [ "$zdmax" != "" ]
124 then
125 echo "# Maximum Zenith Distance: "$zdmax" degree"
126 fi
127 if [ "$thmax" != "" ]
128 then
129 echo "# Maximum Trigger Threshold: "$thmax" DAC counts"
130 fi
131 if [ "$light" != "" ]
132 then
133 echo "# Light Condition Cut: "$lightcut
134 fi
135 if [ "$dust" != "" ]
136 then
137 echo "# Calima Cut: dust < "$dust" ug/cm3"
138 fi
139 if [ "$usedch" == "yes" ]
140 then
141 echo "# Data quality selection was applied. "
142 if [ "$dch" == "" ]
143 then
144 echo "# Standard data check based on cosmic-ray rate: "$dchstd
145 else
146 echo "# You selected a custom datacheck: "$dch
147 fi
148 fi
149 fi
150 echo "#"
151}
152
153function get_results()
154{
155 # some query parts
156
157 # these values use the Crab spectrum from HESS 2006
158 # hess 2006 http://www.aanda.org/articles/aa/pdf/2006/39/aa5351-06.pdf
159 # PL.SetParameter(1,-2.63);
160 # PL.SetParameter(0, 3.45e-11);
161 case "$table" in
162 "AnalysisResultsRunISDC")
163 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.2)"
164 thfactor="(1.18-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00062)"
165 # ETh: 1070 GeV
166 crabflux="1.89"
167 ;;
168 "AnalysisResultsRunCutsLC")
169 zdfactor="pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.5)"
170 thfactor="(1.37-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00118)"
171 # ETh: 785 GeV
172 crabflux="3.14"
173 ;;
174 "AnalysisResultsRunLP")
175 zdfactor="(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))"
176 #zdfactor="(1/(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5)))"
177 thfactor="(1-0.00124/1.21*(fThresholdMinSet-500)*(fThresholdMinSet>=500))"
178 # ETh: 1100 GeV
179 crabflux="1.81"
180 ;;
181 "*")
182 zdfactor="0"
183 thfactor="0"
184 crabflux="0"
185 ;;
186 esac
187
188 # conversion from crab units to fluxes (if a different value is given in setup
189 if [ "$crabfluxconv" != "" ]
190 then
191 crabflux=$crabfluxconv #e-11
192 fi
193 #echo "crabflux: "$crabflux
194 fluxprec=2
195
196 # some names and definitions needed several times below
197 # ontime
198 ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
199 ontime2=" fOnTimeAfterCuts "
200 ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
201 # zd and threshold
202 zenith="fZenithDistance"
203 thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
204 # correction for zd- and th-dependence
205 correvts=" fNumExcEvts/"$zdfactor"/"$thfactor
206 # conversion to CU - stored in each result-table (determined by DD Feb 2019)
207 cufactor="fCU"
208 # some calculations
209 excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
210 CU="SUM(fNumExcEvts/"$cufactor")/SUM("$ontimeif")*3600"
211 CUerr=$excerr"/SUM("$ontimeif")*3600*SUM(fNumExcEvts/"$cufactor")/SUM(fNumExcEvts)"
212 CUcor="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
213 CUcorerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
214 excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
215 CU2="SUM((o.sigevts-o.bgevts)/o.cufactor)/SUM(o.ot)*3600"
216 CUerr2=$excerr2"/SUM(o.ot)*3600*SUM((o.sigevts-o.bgevts)/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
217 CUcor2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
218 CUcorerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
219
220 # columns to be selected
221 # for night-binning
222 ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
223 excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
224 significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
225 numexc="Sum(fNumExcEvts) AS numexc"
226 numsig="Sum(fNumSigEvts) AS numsig"
227 numbg="Sum(fNumBgEvts) AS numbg"
228 excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
229 correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
230 correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
231 cu=" ROUND("$CU", 2) AS cu"
232 cuerr=" ROUND("$CUerr", 2) AS cuerr"
233 cucor=" ROUND("$CUcor", 2) AS cucor"
234 cucorerr=" ROUND("$CUcorerr", 2) AS cucorerr"
235 flux="ROUND("$CU" * "$crabflux", 2) AS flux"
236 fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
237 fluxcor="ROUND("$CUcor" * "$crabflux", 2) AS fluxcor"
238 fluxcorerr="ROUND("$CUcorerr" * "$crabflux", 2) AS fluxcorerr"
239 # for minute binning
240 ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
241 excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
242 significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
243 numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
244 numsig2="Sum(o.sigevts) AS numsig"
245 numbg2="Sum(o.bgevts) AS numbg"
246 excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
247 correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
248 correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
249 cu2=" ROUND("$CU2", 2) AS cu"
250 cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
251 cucor2=" ROUND("$CUcor2", 2) AS cucor"
252 cucorerr2=" ROUND("$CUcorerr2", 2) AS cucorerr"
253 flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
254 fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
255 fluxcor2="ROUND("$CUcor2" * "$crabflux", "$fluxprec") AS fluxcor"
256 fluxcorerr2="ROUND("$CUcorerr2" *"$crabflux", "$fluxprec") AS fluxcorerr"
257
258 case $timeunit in
259 mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
260 start=" Mjd(MIN(fRunStart)) AS start"
261 stop=" Mjd(MAX(fRunStop)) AS stop"
262 deltat=$delta" AS deltat"
263 time=" Mjd(MIN(fRunStart))+"$delta" AS time"
264 delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
265 start2=" Mjd(MIN(o.start)) AS start"
266 stop2=" Mjd(MAX(o.stop)) AS stop"
267 deltat2=$delta2" AS deltat"
268 time2=" Mjd(MIN(o.start))+"$delta2" AS time"
269 ;;
270 unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
271 start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
272 stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
273 deltat=$delta" AS deltat"
274 time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
275 delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
276 start2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
277 stop2=" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
278 deltat2=$delta2" AS deltat"
279 time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
280 ;;
281 *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
282 start=" MIN(fRunStart) AS start"
283 stop=" MAX(fRunStop) AS stop"
284 deltat=$delta" AS deltat"
285 time=" addtime(MIN(fRunStart), "$delta") AS time"
286 delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
287 start2=" MIN(o.start) AS start"
288 stop2=" MAX(o.stop) AS stop"
289 deltat2=$delta2" AS deltat"
290 time2=" addtime(MIN(o.start), "$delta2") AS time"
291 ;;
292 esac
293
294 # from and join of query
295 from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
296
297 # data check based on artificial trigger rate
298 # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
299 #dch=" AND fR750Cor/fR750Ref >0.93 "
300 dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
301
302 # put together where-clause of query
303 # time range and source
304 where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
305 where=$where" AND NOT ISNULL(fNumExcEvts) "
306 # some sanity checks
307 where=$where" AND fRunTypeKey=1 "
308 # zd cut
309 if [ "$zdmax" != "" ]
310 then
311 where=$where" AND fZenithDistanceMax < "$zdmax
312 fi
313 # th cut
314 if [ "$thmax" != "" ]
315 then
316 where=$where" AND "$thresh" < "$thmax
317 fi
318 # dust cut
319 if [ "$dust" != "" ]
320 then
321 where=$where" AND fTNGDust<"$dust
322 fi
323 # light condition cut
324 if [ "$light" == "nomoon" ]
325 then
326 lightcut=" fZenithDistanceMoon>90"
327 fi
328 if [ "$light" == "dark" ]
329 then
330 lightcut=" fMoonZenithDistance>90 AND fSunZenithDistance>108 "
331 fi
332 if [ "$light" != "" ]
333 then
334 where=$where" AND "$lightcut
335 fi
336 querybase=$from$where
337
338 if [ "$usedch" == "yes" ]
339 then
340 if [ "$dch" == "" ]
341 then
342 querydch=$dchstd
343 else
344 echo "you are using for datacheck: "$dch
345 querydch=$dch
346 fi
347 else
348 querydch=
349 fi
350
351
352 if [ $bin -le 0 ]
353 then
354 # first part of the query
355 querystart="SELECT "
356 querystart=$querystart" "$time", "$start", "$stop", "
357 # final part of the query
358 if [ $bin -eq 0 ]
359 then
360 orderby=" fPeriod "
361 #querystart=$querystart" fPeriod AS num, "
362 queryend=" GROUP BY fPeriod "
363 else
364 num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
365 orderby=$num
366 #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
367 queryend=" GROUP BY "$num
368 fi
369 #queryend=" GROUP BY num "
370 if [ "$ontimelimit" = "" ]
371 then
372 queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
373 else
374 queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
375 fi
376
377 # internal
378 queryint=$querystart
379 queryint=$queryint" "$excrate", "$cu", "$flux", "$correxcrate", "$cucor", "$fluxcor", "
380 queryint=$queryint" "$deltat", "$ontime", "
381 queryint=$queryint" "$excrateerr", "$cuerr", "$fluxerr", "$correxcrateerr", "$cucorerr", "$fluxcorerr", "
382 queryint=$queryint" "$significance", "
383 queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
384 queryint=$queryint" "$numexc", "$numsig", "$numbg", "
385 queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
386 queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
387 queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
388 queryint=$queryint" "$querybase" "$querydch" "$queryend
389
390 # for collaborators
391 querycol=$querystart
392 querycol=$querycol" "$excrate", "$correxcrate", "$cucor", "$fluxcor", "
393 querycol=$querycol" "$deltat", "$ontime", "
394 querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cucorerr", "$fluxcorerr", "
395 querycol=$querycol" "$significance
396 querycol=$querycol" "$querybase" "$querydch" "$queryend
397
398 # external
399 # no datacheck applied for external files
400 queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
401
402 else
403 # first part of the query
404 querystart="SELECT "
405 querystart=$querystart" "$time2", "$start2", "$stop2", "
406
407 # final part of the query
408 querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
409 querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
410 querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
411 querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
412 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
413 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
414 querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
415 querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
416
417 # internal
418 queryint=$querystart
419 queryint=$queryint" "$excrate2", "$cu2", "$flux2", "$correxcrate2", "$cucor2", "$fluxcor2", "
420 queryint=$queryint" "$deltat2", "$ontime2", "
421 queryint=$queryint" "$excrateerr2", "$cuerr2", "$fluxerr2", "$correxcrateerr2", "$cucorerr2", "$fluxcorerr2", "
422 queryint=$queryint" "$significance2", "
423 queryint=$queryint" avg(o.night) AS night, "
424 queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
425 queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
426 queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
427 queryint=$queryint" "$querybase
428
429 # for collaborators
430 querycol=$querystart
431 querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$fluxcor2", "
432 querycol=$querycol" "$deltat2", "$ontime2", "
433 querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxcorerr2", "
434 querycol=$querycol" "$significance2
435 querycol=$querycol" "$querybase
436
437 # external
438 queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
439
440 fi
441
442 # write file for externals only for allowed binnings
443 if [ $bin -eq 20 ] || [ $bin -eq -1 ]
444 then
445 fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
446 if [ "$overwrite" = "yes" ]
447 then
448 if [ "$mode" != "auto" ]
449 then
450 echo "creating "$fileext" ..."
451 fi
452 echo "# This file was created at "`date` > $fileext
453 print_policy >> $fileext
454 fi
455 print_selection >> $fileext
456 headerext="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] (stop-start)/2["$timeunit"] excess-rate_error[evts/h] "
457 echo $headerext >> $fileext
458 #echo "$queryext"
459 mysql --defaults-file=$sqlpw -s -e "$queryext" >> $fileext
460 #mysql --defaults-file=$sqlpw -e "$queryext"
461 fi
462 if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
463 then
464 return
465 fi
466
467 fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
468 if [ "$overwrite" = "yes" ]
469 then
470 if [ "$mode" != "auto" ]
471 then
472 echo "creating "$fileint" ..."
473 fi
474 echo "# This file was created at "`date` > $fileint
475 print_policy >> $fileint
476 fi
477 print_selection >> $fileint
478 echo "# The following query was used: " >> $fileint
479 echo "# $queryint" >> $fileint
480 echo "#" >> $fileint
481 headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] flux[CU] flux[e-11ph/cm2/s] corrected_excess-rate[evts/h] corrected_flux[CU] corrected_flux[e-11ph/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
482 headerint=$headerint" excess-rate_error[evts/h] flux_error[CU] flux_error[e-11ph/cm2/s] corrected_excess-rate_error[evts/h] corrected_flux_error[CU] corrected_flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
483 headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
484 echo $headerint >> $fileint
485 #echo "$queryint"
486 mysql --defaults-file=$sqlpw -s -e "$queryint" >> $fileint
487 #mysql --defaults-file=$sqlpw -e "$queryint"
488 if [ "$mode" == "auto" ]
489 then
490 return
491 fi
492
493 filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
494 if [ "$overwrite" = "yes" ]
495 then
496 echo "creating "$filecol" ..."
497 echo "# This file was created at "`date` > $filecol
498 print_policy >> $filecol
499 fi
500 print_selection >> $filecol
501 #echo "# The following query was used: " >> $filecol
502 #echo "# $querycol" >> $filecol
503 #echo "#" >> $filecol
504 headercol="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] corrected_flux[CU] corrected_flux[e-11ph/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
505 headercol=$headercol" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] corrected_flux_error[CU] corrected_flux_error[e-11ph/cm2/s] significance "
506 echo $headercol >> $filecol
507 #echo "$querycol"
508 mysql --defaults-file=$sqlpw -s -e "$querycol" >> $filecol
509 #mysql --defaults-file=$sqlpw -e "$querycol
510}
511
512# evaluation of command line options (for usage with download.php)
513
514if [ ${#@} -eq 13 ]
515then
516 #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
517 mode="auto"
518 overwrite="yes"
519 # setup for usage with download.php
520 datapath="/home/factwww/dch/data"
521 #datapath="./data"
522 sqlpw=/home/fact/.mysql.pw
523 #sqlpw=/home/fact/.mysql.pw2
524 #host=10.0.100.21
525 #dbname=factdata
526 nightmin=$1
527 nightmax=$2
528 source=$3
529 bin=$4
530 if [ "$bin" == "00" ]
531 then
532 bin=0
533 fi
534 email=$5
535 table=$6
536 timeunit=$7
537 expert=$8
538 usedch=$9 # novalue gives same result as no
539 if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
540 then
541 zdmax=${10}
542 fi
543 if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
544 then
545 thmax=${11}
546 fi
547 if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
548 then
549 light=${12}
550 fi
551 if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
552 then
553 dust=${13}
554 fi
555 name=`echo $email | sed -e 's/@/-at-/'`
556 get_results
557
558 # sending email
559 if [ "$expert" == "yes" ]
560 then
561 cat $fileint | mail -s 'FACT internal data download' -b qla@fact-project.org -r qla@fact-project.org $email
562 else
563 cat $fileext | mail -s 'FACT data download' -b qla@fact-project.org -r qla@fact-project.org $email
564 fi
565
566 exit
567fi
568
569
570
571# -------------------------------------------------------------------------------------- #
572# SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
573# -------------------------------------------------------------------------------------- #
574# #
575# The lines below define the basic setup for the database and give examples and #
576# explanations for the various options available. #
577# The request of the data itself is done with a smaller setup further down. #
578# #
579# -------------------------------------------------------------------------------------- #
580#
581# ----------
582# DB SETUP
583# ----------
584# path to file with mysql setup (user, password, host, db-name)
585sqlpw=/home/$USER/.mysql.pw.local
586sqlpw=/home/$USER/.mysql.pw.ethz.fact
587#
588# -------------
589# BASIC SETUP
590# -------------
591# output path
592path=`dirname $0`
593datapath=$path"/data"
594# create directory for data files
595if ! [ -e $datapath ]
596then
597 mkdir $datapath
598fi
599# time unit
600#timeunit=timestamp # default
601#timeunit=unix
602timeunit=mjd
603# time binning
604# positive values: minutes
605# negative values: days
606# special case 0: period
607# for season binning choose -365 and according start date
608#bin=20 # minutes
609#bin=0 # period
610bin=-1 # nightly
611#bin=-365 # yearly
612# choose analysis
613#table="AnalysisResultsAllQLA" # N/A
614#table="AnalysisResultsRunLP" # QLA
615#table="AnalysisResultsRunISDC" # ISDC
616table="AnalysisResultsRunCutsLC" # CutsLC
617# time range
618nightmin=20111115
619nightmax=20201231
620# overwrite dataset file?
621# (useful to combine different binnings in one file -> set to "no")
622overwrite="yes"
623# optional: require minimal ontime per bin (default 20 min)
624#ontimelimit=30 # 30 min
625ontimelimit= # default 20 min
626# data quality selection
627# if you explicitely don't want a datacheck, you can comment the following line
628usedch="yes"
629# use your own datacheck instead
630# use a line like the following defining your own data quality selection cut
631#dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
632# apply additional predefined cuts
633# light conditions
634#light="nomoon" # only data with no moon (but twilight allowed)
635#light="dark" # only dark night data
636# TNG dust - cut away data with calima
637#dust=1
638#dust=10
639# use different conversion from CU to fluxes
640#crabfluxconv="2.5"
641
642
643# -------------------------------------------------------------------------------------- #
644# SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
645# -------------------------------------------------------------------------------------- #
646# #
647# Adapt the lines below to your needs. #
648# Overwrite default settings above. #
649# The data-request is sent with the line 'get_results'. #
650# Minumum setup: Define source key and name for file. #
651# The list of source keys can be found at #
652# https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
653# More examples can be found further down. #
654# #
655# -------------------------------------------------------------------------------------- #
656
657# some test
658table="AnalysisResultsRunCutsLC" # CutsLC
659bin=0
660# Mrk 501
661source=2
662name="Mrk501_all_P"
663get_results
664
665#exit
666
667# LC for ICRC
668table="AnalysisResultsRunCutsLC" # CutsLC
669bin=-1
670# Mrk 421
671source=1
672name="Mrk421_all_nightly"
673get_results
674# Mrk 501
675source=2
676name="Mrk501_all_nightly"
677get_results
678# 1959
679source=7
680name="1959_all_nightly"
681get_results
682
683# 2344
684source=3
685name="2344_all_nightly"
686get_results
687
688# Crab
689source=5
690name="Crab_all_nightly"
691get_results
692
693
694#exit
695
696# Mrk 501
697source=2
698zdmax=90
699thmax=1500
700# new analysis
701table="AnalysisResultsRunCutsLC" # CutsLC
702nightmin=20140520
703nightmax=20140930
704name="Mrk501_nightly_newAnalysis_forHESS"
705bin=-1
706get_results
707name="Mrk501_7d_newAnalysis_forHESS"
708bin=-7
709get_results
710nightmin=20140623
711nightmax=20140623
712name="Mrk501_5min_FlareNight_newAnalysis_forHESS"
713bin=5
714get_results
715nightmin=20140622
716nightmax=20140624
717name="Mrk501_5min_Flare_newAnalysis_forHESS"
718bin=5
719get_results
720
721exit
722
723# LC for INTEGRAL Proposal
724table="AnalysisResultsRunCutsLC" # CutsLC
725bin=-1
726# Mrk 421
727source=1
728name="Mrk421_all_nightly"
729get_results
730# Mrk 501
731source=2
732name="Mrk501_all_nightly"
733get_results
734# 1959
735source=7
736name="1959_all_nightly"
737get_results
738# for other INTEGRAL proposal
739# Mrk 421
740bin=0
741source=1
742name="Mrk421_all_period"
743get_results
744
745exit
746
747
748# Crab
749source=5
750nightmin=20121214
751nightmax=20180418
752table="AnalysisResultsRunCutsLC" # CutsLC
753bin=-1
754zdmax=30
755thmax=330
756name="testCrabnightlycheck"
757get_results
758
759exit
760
761# Crab
762source=5
763nightmin=20121214
764nightmax=20180418
765table="AnalysisResultsRunCutsLC" # CutsLC
766bin=20
767name="testCrab20min"
768get_results
769
770exit
771
772# Crab
773source=5
774nightmin=20121214
775nightmax=20180418
776table="AnalysisResultsRunCutsLC" # CutsLC
777bin=-1
778name="testCrabnightly"
779get_results
780
781exit
782
783# Crab
784source=5
785#nightmin=20121214
786#nightmax=20180418
787table="AnalysisResultsRunCutsLC" # CutsLC
788bin=-1
789name="testCraball"
790get_results
791
792exit
793
794
795# Crab
796source=5
797nightmin=20121214
798nightmax=20180418
799table="AnalysisResultsRunCutsLC" # CutsLC
800bin=60
801name="testCrab"
802get_results
803
804exit
805
806
807# for 2344 paper with MAGIC
808
809source=3
810zdmax=90
811thmax=1500
812
813# new analysis
814table="AnalysisResultsRunCutsLC" # CutsLC
815nightmin=20160618
816nightmax=20160815
817name="2344_2016flare_newAnalysis"
818bin=-7
819get_results
820overwrite="no"
821bin=-28
822nightmin=20160816
823nightmax=20161031
824get_results
825overwrite="yes"
826
827# ISDC analysis
828table="AnalysisResultsRunISDC"
829nightmin=20160618
830nightmax=20160815
831name="2344_2016flare_stdAnalysis"
832bin=-7
833get_results
834overwrite="no"
835bin=-28
836nightmin=20160816
837nightmax=20161031
838get_results
839overwrite="yes"
840
841
842usedch="no"
843# new analysis
844table="AnalysisResultsRunCutsLC" # CutsLC
845nightmin=20160618
846nightmax=20160815
847name="2344_2016flare_newAnalysis_noDCh"
848bin=-7
849get_results
850overwrite="no"
851bin=-28
852nightmin=20160816
853nightmax=20161031
854get_results
855overwrite="yes"
856
857# ISDC analysis
858table="AnalysisResultsRunISDC"
859nightmin=20160618
860nightmax=20160815
861name="2344_2016flare_stdAnalysis_noDCh"
862bin=-7
863get_results
864overwrite="no"
865bin=-28
866nightmin=20160816
867nightmax=20161031
868get_results
869overwrite="yes"
870
871usedch="yes"
872
873
874exit
875
876# Mrk 501
877source=2
878zdmax=90
879thmax=1500
880# new analysis
881table="AnalysisResultsRunCutsLC" # CutsLC
882nightmin=20140520
883nightmax=20140930
884name="Mrk501_nightly_newAnalysis_forHESS"
885bin=-1
886get_results
887name="Mrk501_7d_newAnalysis_forHESS"
888bin=-7
889get_results
890nightmin=20140623
891nightmax=20140623
892name="Mrk501_5min_FlareNight_newAnalysis_forHESS"
893bin=5
894get_results
895# isdc analysis
896thmax=850
897table="AnalysisResultsRunISDC"
898nightmin=20140520
899nightmax=20140930
900name="Mrk501_nightly_stdAnalysis_forHESS"
901bin=-1
902get_results
903name="Mrk501_7d_stdAnalysis_forHESS"
904bin=-7
905get_results
906nightmin=20140623
907nightmax=20140623
908name="Mrk501_5min_FlareNight_stdAnalysis_forHESS"
909bin=5
910get_results
911# qla
912thmax=500
913table="AnalysisResultsRunLP"
914nightmin=20140520
915nightmax=20140930
916name="Mrk501_nightly_QLA_forHESS"
917bin=-1
918get_results
919
920exit
921
922# 501 MAGIC
923source=2
924name="Mrk501_2014_forMAGIC"
925bin=-1
926crabfluxconv="2.87" # using threshold of 830GeV
927crabfluxconv="3.01" # using threshold of 830GeV and MAGIC Crab spectrum
928nightmin=20140714
929nightmax=20140805
930get_results
931
932name="Mrk501_2014_forMAGIC_20min"
933bin=20
934get_results
935
936
937exit
938
939bin=30
940name="Mrk501_2014_forMAGIC30"
941get_results
942
943bin=0
944name="P"
945nightmin=20140501
946nightmax=20140930
947get_results
948
949bin=20
950nightmin=20140623
951nightmax=20140623
952name="Mrk501_test"
953get_results
954
955
956# end script here
957exit
958
959
960
961#
962# more examples
963#
964
965# Mrk 421
966source=1
967name="Mrk421_nightly"
968bin=-1
969get_results
970name="Mrk421_20min"
971bin=20
972get_results
973name="Mrk421_3d"
974bin=-3
975get_results
976name="Mrk421_10d"
977bin=-10
978get_results
979name="Mrk421_period"
980bin=0
981get_results
982
983
984
985# Mrk 501
986source=2
987name="Mrk501_nightly"
988bin=-1
989get_results
990name="Mrk501_20min"
991bin=20
992get_results
993name="Mrk501_3d"
994bin=-3
995get_results
996name="Mrk501_10d"
997bin=-10
998get_results
999name="Mrk501_period"
1000bin=0
1001get_results
1002
1003
1004
1005# 2344
1006source=3
1007name="2344_nightly"
1008bin=-1
1009get_results
1010name="2344_20min"
1011bin=20
1012get_results
1013name="2344_period"
1014bin=0
1015get_results
1016
1017
1018
1019# 1959
1020source=7
1021name="1959_nightly"
1022bin=-1
1023get_results
1024name="1959_20min"
1025bin=20
1026get_results
1027name="1959_period"
1028bin=0
1029get_results
1030
1031
1032
1033# 0323
1034source=12
1035name="0323_nightly"
1036bin=-1
1037get_results
1038name="0323_20min"
1039bin=20
1040get_results
1041name="0323_period"
1042bin=0
1043get_results
1044
1045
1046
1047# crab
1048source=5
1049name="Crab_nightly"
1050bin=-1
1051get_results
1052name="Crab_20min"
1053bin=20
1054get_results
1055name="Crab_period"
1056bin=0
1057get_results
1058name="Crab_season"
1059bin=-365
1060nightmin=20110716
1061nightmax=20180716
1062get_results
1063
1064
1065
1066name="1959_2016"
1067source=7
1068bin=-1
1069nightmin=20160201
1070nightmax=20161105
1071get_results
1072
1073name="1959_all_variable"
1074overwrite="no"
1075source=7
1076bin=-365
1077nightmin=20120201
1078nightmax=20130131
1079get_results
1080nightmin=20130201
1081nightmax=20140131
1082get_results
1083nightmin=20140201
1084nightmax=20150131
1085get_results
1086bin=0
1087nightmin=20150201
1088nightmax=20160131
1089get_results
1090bin=-1
1091nightmin=20160201
1092nightmax=20170131
1093get_results
1094bin=0
1095nightmin=20170201
1096nightmax=20180131
1097get_results
1098
1099
1100
1101overwrite="yes"
1102name="1959_all_variable2"
1103overwrite="no"
1104source=7
1105bin=-365
1106nightmin=20120201
1107nightmax=20130131
1108get_results
1109nightmin=20130201
1110nightmax=20140131
1111get_results
1112nightmin=20140201
1113nightmax=20150131
1114get_results
1115bin=0
1116nightmin=20150201
1117nightmax=20160131
1118get_results
1119bin=-1
1120nightmin=20160201
1121nightmax=20160817
1122get_results
1123bin=0
1124nightmin=20160818
1125nightmax=20180131
1126get_results
1127
1128
1129
1130overwrite="yes"
1131bin=0
1132source=3
1133name="2344period"
1134get_results
1135
1136
1137
1138# flare night (HESS)
1139name="Mrk501_10min_flarenight"
1140source=2
1141bin=10
1142nightmin=20140623
1143nightmax=20140623
1144get_results
1145
1146
1147
1148# flare night (HESS)
1149name="Mrk501_5min_flarenight"
1150source=2
1151bin=5
1152nightmin=20140623
1153nightmax=20140623
1154get_results
1155
1156
1157
1158
1159# full sample
1160name="Mrk421_all_nightly"
1161source=1
1162get_results
1163
1164name="Mrk501_all_nightly"
1165source=2
1166get_results
1167
1168name="1959_all_nightly"
1169source=7
1170get_results
1171
1172name="2344_all_nightly"
1173source=3
1174get_results
1175
1176
1177
1178name="HESE20160427"
1179source=19
1180nightmin=20160425
1181bin=-10
1182get_results
1183
1184name="AMON20160731"
1185source=21
1186nightmin=20160730
1187bin=-10
1188get_results
1189
1190
1191
Note: See TracBrowser for help on using the repository browser.