source: trunk/DataCheck/Tools/get_data.sh@ 19272

Last change on this file since 19272 was 19066, checked in by Daniela Dorner, 6 years ago
updated policy, headers and email-adresses
  • Property svn:executable set to *
File size: 27.4 KB
Line 
1#!/bin/bash
2
3# ---------------------------------------------------------------- #
4# README README README README README README README README README #
5# ---------------------------------------------------------------- #
6# #
7# To use this script, you need #
8# - a computer with access to the FACT database in La Palma #
9# - a file with the password of a valid mysql-user #
10# - to define the setup below for #
11# a) the DB access #
12# b) the data you want to have #
13# #
14# To define the setup, search for SETUP in this script and #
15# read the details there #
16# #
17# Per data request, you get up to 3 files: #
18# *_internal.dat #
19# *_collaborators.dat #
20# *_external.dat (only if binning is 20min or nightly) #
21# #
22# Please have in mind that this started as a tool for myself, then #
23# others started using it. Also the script is not yet finalized. #
24# In case you find problems and/or have a feature request, please #
25# send and email to dorner@astro.uni-wuerzburg.de #
26# #
27# ---------------------------------------------------------------- #
28# README README README README README README README README README #
29# ---------------------------------------------------------------- #
30
31
32
33
34
35# ToDo (notes DD):
36# ----------------
37# - finalize policy and references
38# must-have
39# - update function for zd/th-correction
40# - update CU for QLA
41# - add CU for ISDC analysis
42# - add < 20121212 data for QLA
43# - check crab flux
44# nice-to-have
45# - add E2dNdE
46# - functionality to determine start time for seaon-binning
47# - offer several predefined datachecks?
48# - file as attachment to email ofr download.php
49
50#
51# content of files (wish list):
52# -----------------------------
53# REMARK: keep order of columns to allow for reading with TGraph directly from file: X Y EX EY
54#
55# internal
56# --------
57# time: time, delta time, start, stop, ontime
58# flux: excrate, excerr, corrate, corerr, CU CUerr, flux, fluxerr,
59# other info on flux: signif, cu-factor, num exc, num sig, num bg
60# other info: zd th R750cor R750ref
61#
62# external (allow only 20min and nightly binning)
63# --------
64# time: time, delta time, start, stop
65# flux: excrate, excerr
66#
67# collaborators
68# -------------
69# time: time, delta time, start, stop, ontime
70# flux: excrate, excerr, corrate, corerr, flux, flux-err, significance
71#
72# additional information to put:
73# ------------------------------
74# timestamp of creation
75# query (for debugging / answering questions)
76# policy (adapted for internal/collaborators/external) [define in files to be used also by Send_Data*.sh
77#
78
79
80function print_policy()
81{
82 echo "#"
83 echo "# Data Usage Policy: "
84 if [ "$expert" == "no" ]
85 then
86 echo "# Using data from the FACT Quick Look Analysis, you agree to cite the FACT design"
87 echo "# paper (H. Anderhub et al. JINST 8 P6008) and the quick look analysis website "
88 echo "# (https://fact-project.org/monitoring)."
89 echo "#"
90 echo "# References: "
91 echo "# FACT design paper: http://adsabs.harvard.edu/abs/2013JInst...8P6008A"
92 echo "# http://iopscience.iop.org/1748-0221/8/06/P06008 "
93 echo "# FACT Performance Paper: http://adsabs.harvard.edu/abs/2014JInst...9P0012B"
94 echo "# http://iopscience.iop.org/1748-0221/9/10/P10012"
95 echo "# FACT quick look analysis: https://fact-project.org/monitoring"
96 echo "# http://adsabs.harvard.edu/abs/2015arXiv150202582D"
97 echo "# If you intend to use data or information from this website, please let us know for reference."
98 else
99 echo "# As a member or associated member of the FACT Collaboration, you have access to internal information."
100 echo "# Any publication using FACT internal information has to have the full FACT author list."
101 fi
102 echo "#"
103}
104
105print_selection()
106{
107 echo "#"
108 echo "# Your Selection: "
109 sourcename=`mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "SELECT fSourceName FROM Source WHERE fSourceKey="$source`
110 echo "# Source: "$sourcename
111 echo "# Time range: "$nightmin"-"$nightmax
112 echo "# Time format: "$timeunit
113 if [ $bin -lt 0 ]
114 then
115 unit="night(s)"
116 else
117 if [ $bin -eq 0 ]
118 then
119 unit="periods"
120 else
121 unit="minutes"
122 fi
123 fi
124 binning=`echo $bin | sed -e 's/-//'`" "$unit
125 echo "# Binning: "$binning
126 if [ "$expert" = "yes" ]
127 then
128 echo "# Additional Internal Selection: "
129 echo "# "`echo $table | sed -e 's/AnalysisResultsRun//'`"-Analysis was used."
130 if [ "$zdmax" != "" ]
131 then
132 echo "# Maximum Zenith Distance: "$zdmax" degree"
133 fi
134 if [ "$thmax" != "" ]
135 then
136 echo "# Maximum Trigger Threshold: "$thmax" DAC counts"
137 fi
138 if [ "$light" != "" ]
139 then
140 echo "# Light Condition Cut: "$lightcut
141 fi
142 if [ "$dust" != "" ]
143 then
144 echo "# Calima Cut: dust < "$dust" ug/cm3"
145 fi
146 if [ "$usedch" == "yes" ]
147 then
148 echo "# Data quality selection based on the cosmic ray rate was applied."
149 fi
150 fi
151 echo "#"
152}
153
154function get_results()
155{
156 # some query parts
157
158 # some numbers for flux calculation
159 crabflux="3.37e-11"
160 fluxprec=13
161 crabflux="3.37"
162 fluxprec=2
163
164 # some names and definitions needed several times below
165 # ontime
166 ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
167 ontime2=" fOnTimeAfterCuts "
168 ontimeif=" IF(ISNULL(fEffectiveOn), "$ontime2", "$ontime1") "
169 # zd and threshold
170 zenith="fZenithDistance"
171 thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
172 # correction factor for excess rate (formula by TB)
173 correvts=" fNumExcEvts*(pow(cos("$zenith"Mean*PI()/180),3)+14.8/21.9*pow(sin(2*"$zenith"Mean*PI()/180),5))/((1-0.00124/1.21*("$thresh"-500)*("$thresh">=500))) "
174 # conversion to CU (determined by DD for QLA)
175 # https://www.fact-project.org/logbook/showthread.php?tid=4927
176 cufactor="CUQLA(fNight)" # missing: ISDC analysis
177 # some calculations
178 excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
179 CU="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
180 CUerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
181 excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
182 CU2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
183 CUerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
184
185 # columns to be selected
186 # for night-binning
187 ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
188 excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
189 significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
190 numexc="Sum(fNumExcEvts) AS numexc"
191 numsig="Sum(fNumSigEvts) AS numsig"
192 numbg="Sum(fNumBgEvts) AS numbg"
193 excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
194 correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
195 correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
196 cu=" ROUND("$CU", 2) AS cu"
197 cuerr=" ROUND("$CUerr", 2) AS cuerr"
198 flux="ROUND("$CU" * "$crabflux", 2) AS flux"
199 fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
200 # for minute binning
201 ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
202 excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
203 significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
204 numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
205 numsig2="Sum(o.sigevts) AS numsig"
206 numbg2="Sum(o.bgevts) AS numbg"
207 excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
208 correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
209 correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
210 cu2=" ROUND("$CU2", 2) AS cu"
211 cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
212 flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
213 fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
214
215 case $timeunit in
216 mjd) delta="(Mjd(MAX(fRunStop))-Mjd(MIN(fRunStart)))/2"
217 start=" Mjd(MIN(fRunStart)) AS start"
218 stop=" Mjd(MAX(fRunStop)) AS stop"
219 deltat=$delta" AS deltat"
220 time=" Mjd(MIN(fRunStart))+"$delta" AS time"
221 delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
222 start2=" Mjd(MIN(o.start)) AS start"
223 stop2=" Mjd(MAX(o.stop)) AS stop"
224 deltat2=$delta2" AS deltat"
225 time2=" Mjd(MIN(o.start))+"$delta2" AS time"
226 ;;
227 unix) delta="(Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')))/2"
228 start="Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM')) AS start"
229 stop="Unix_timestamp(CONVERT_TZ(MAX(fRunStop), '+00:00', 'SYSTEM')) AS stop"
230 deltat=$delta" AS deltat"
231 time=" Unix_timestamp(CONVERT_TZ(MIN(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
232 delta2="(Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')))/2"
233 start2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
234 stop2=" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
235 deltat2=$delta2" AS deltat"
236 time2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
237 ;;
238 *) delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), MIN(fRunStart)))/2)"
239 start=" MIN(fRunStart) AS start"
240 stop=" MAX(fRunStop) AS stop"
241 deltat=$delta" AS deltat"
242 time=" addtime(MIN(fRunStart), "$delta") AS time"
243 delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), MIN(o.start)))/2)"
244 start2=" MIN(o.start) AS start"
245 stop2=" MAX(o.stop) AS stop"
246 deltat2=$delta2" AS deltat"
247 time2=" addtime(MIN(o.start), "$delta2") AS time"
248 ;;
249 esac
250
251 # from and join of query
252 from=" FROM RunInfo LEFT JOIN "$table" USING (fNight, fRunID) "
253
254 # data check based on artificial trigger rate
255 # details see https://www.fact-project.org/logbook/showthread.php?tid=5790
256 #dch=" AND fR750Cor/fR750Ref >0.93 "
257 dchstd=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
258
259 # put together where-clause of query
260 # time range and source
261 where=" WHERE fSourceKey="$source" AND fNight BETWEEN "$nightmin" AND "$nightmax
262 where=$where" AND NOT ISNULL(fNumExcEvts) "
263 # some sanity checks
264 where=$where" AND fRunTypeKey=1 "
265 # zd cut
266 if [ "$zdmax" != "" ]
267 then
268 where=$where" AND fZenithDistanceMax < "$zdmax
269 fi
270 # th cut
271 if [ "$thmax" != "" ]
272 then
273 where=$where" AND "$thresh" < "$thmax
274 fi
275 # dust cut
276 if [ "$dust" != "" ]
277 then
278 where=$where" AND fTNGDust<"$dust
279 fi
280 # light condition cut
281 if [ "$light" == "nomoon" ]
282 then
283 lightcut=" fZenithDistanceMoon>90"
284 fi
285 if [ "$light" == "dark" ]
286 then
287 lightcut=" fMoonZenithDistance>90 AND fSunZenithDistance>108 "
288 fi
289 if [ "$light" != "" ]
290 then
291 where=$where" AND "$lightcut
292 fi
293 querybase=$from$where
294
295 if [ "$usedch" == "yes" ]
296 then
297 if [ "$dch" == "" ]
298 then
299 querydch=$dchstd
300 else
301 echo "you are using for datacheck: "$dch
302 querydch=$dch
303 fi
304 fi
305
306
307 if [ $bin -le 0 ]
308 then
309 # first part of the query
310 querystart="SELECT "
311 querystart=$querystart" "$time", "$start", "$stop", "
312 # final part of the query
313 if [ $bin -eq 0 ]
314 then
315 orderby=" fPeriod "
316 #querystart=$querystart" fPeriod AS num, "
317 queryend=" GROUP BY fPeriod "
318 else
319 num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
320 orderby=$num
321 #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
322 queryend=" GROUP BY "$num
323 fi
324 #queryend=" GROUP BY num "
325 if [ "$ontimelimit" = "" ]
326 then
327 queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
328 else
329 queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
330 fi
331
332 # internal
333 queryint=$querystart
334 queryint=$queryint" "$excrate", "$correxcrate", "$cu", "$flux", "
335 queryint=$queryint" "$deltat", "$ontime", "
336 queryint=$queryint" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
337 queryint=$queryint" "$significance", "
338 queryint=$queryint" MIN(fNight) AS nightmin, MAX(fNight) AS nightmax, "
339 queryint=$queryint" "$numexc", "$numsig", "$numbg", "
340 queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
341 queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
342 queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor, ROUND(AVG(fR750Ref), 2) AS R750ref "
343 queryint=$queryint" "$querybase" "$querydch" "$queryend
344
345 # for collaborators
346 querycol=$querystart
347 querycol=$querycol" "$excrate", "$correxcrate", "$cu", "$flux", "
348 querycol=$querycol" "$deltat", "$ontime", "
349 querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
350 querycol=$querycol" "$significance
351 querycol=$querycol" "$querybase" "$querydch" "$queryend
352
353 # external
354 # no datacheck applied for external files
355 queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
356
357 else
358 # first part of the query
359 querystart="SELECT "
360 querystart=$querystart" "$time2", "$start2", "$stop2", "
361
362 # final part of the query
363 querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
364 querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
365 querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
366 querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
367 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
368 querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
369 querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
370 querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
371
372 # internal
373 queryint=$querystart
374 queryint=$queryint" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
375 queryint=$queryint" "$deltat2", "$ontime2", "
376 queryint=$queryint" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
377 queryint=$queryint" "$significance2", "
378 queryint=$queryint" avg(o.night) AS night, "
379 queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
380 queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
381 queryint=$queryint" ROUND(AVG(o.cufactor), 1) AS cufactor, ROUND(AVG(o.R750cor), 2) AS R750cor, ROUND(AVG(o.R750ref), 2) AS R750ref "
382 queryint=$queryint" "$querybase
383
384 # for collaborators
385 querycol=$querystart
386 querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
387 querycol=$querycol" "$deltat2", "$ontime2", "
388 querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
389 querycol=$querycol" "$significance2
390 querycol=$querycol" "$querybase
391
392 # external
393 queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
394
395 fi
396
397 # write file for externals only for allowed binnings
398 if [ $bin -eq 20 ] || [ $bin -eq -1 ]
399 then
400 fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
401 if [ "$overwrite" = "yes" ]
402 then
403 if [ "$mode" != "auto" ]
404 then
405 echo "creating "$fileext" ..."
406 fi
407 echo "# This file was created at "`date` > $fileext
408 print_policy >> $fileext
409 fi
410 print_selection >> $fileext
411 headerext="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] (stop-start)/2["$timeunit"] excess-rate_error[evts/h] "
412 echo $headerext >> $fileext
413 #echo "$queryext"
414 mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryext" >> $fileext
415 #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryext"
416 fi
417 if [ "$mode" == "auto" ] && [ "$expert" == "no" ]
418 then
419 return
420 fi
421
422 fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
423 if [ "$overwrite" = "yes" ]
424 then
425 if [ "$mode" != "auto" ]
426 then
427 echo "creating "$fileint" ..."
428 fi
429 echo "# This file was created at "`date` > $fileint
430 print_policy >> $fileint
431 print_selection >> $fileint
432 echo "# The following query was used: " >> $fileint
433 echo "# "$queryint >> $fileint
434 echo "#" >> $fileint
435 fi
436 headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
437 headerint=$headerint" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
438 headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
439 echo $headerint >> $fileint
440 #echo "$queryint"
441 mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryint" >> $fileint
442 #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryint"
443 if [ "$mode" == "auto" ]
444 then
445 return
446 fi
447
448 filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
449 if [ "$overwrite" = "yes" ]
450 then
451 echo "creating "$filecol" ..."
452 echo "# This file was created at "`date` > $filecol
453 print_policy >> $filecol
454 print_selection >> $filecol
455 echo "# The following query was used: " >> $filecol
456 echo "# "$querycol >> $filecol
457 echo "#" >> $filecol
458 fi
459 headercol="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] corrected_excess-rate[evts/h] flux[CU] flux[e-11/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
460 headercol=$headercol" excess-rate_error[evts/h] corrected_excess-rate_error[evts/h] flux_error[CU] flux_error[e-11/cm2/s] significance "
461 echo $headercol >> $filecol
462 #echo "$querycol"
463 mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$querycol" >> $filecol
464 #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$querycol
465}
466
467# evaluation of command line options (for usage with download.php)
468
469if [ ${#@} -eq 13 ]
470then
471 #get_data.sh $start $stop $source $timebin $email $table $time $expert $dch $zd $th $light $dust
472 mode="auto"
473 overwrite="yes"
474 # setup
475 datapath="/home/factwww/dch/data"
476 #datapath="./data"
477 sqlpw=/home/fact/.mysql.pw
478 #sqlpw=/home/fact/.mysql.pw2
479 host=10.0.100.21
480 dbname=factdata
481 nightmin=$1
482 nightmax=$2
483 source=$3
484 bin=$4
485 if [ "$bin" == "00" ]
486 then
487 bin=0
488 fi
489 email=$5
490 table=$6
491 timeunit=$7
492 expert=$8
493 usedch=$9 # novalue gives same result as no
494 if [ "${10}" != "novalue" ] && [ "${10}" != "all" ]
495 then
496 zdmax=${10}
497 fi
498 if [ "${11}" != "novalue" ] && [ "${11}" != "all" ]
499 then
500 thmax=${11}
501 fi
502 if [ "${12}" != "novalue" ] && [ "${12}" != "all" ]
503 then
504 light=${12}
505 fi
506 if [ "${13}" != "novalue" ] && [ "${13}" != "all" ]
507 then
508 dust=${13}
509 fi
510 name=`echo $email | sed -e 's/@/-at-/'`
511 get_results
512
513 # sending email
514 if [ "$expert" == "yes" ]
515 then
516 cat $fileint | mail -s 'FACT internal data download' -b qla@fact-project.org -r qla@fact-project.org $email
517 else
518 cat $fileext | mail -s 'FACT data download' -b qla@fact-project.org -r qla@fact-project.org $email
519 fi
520
521 exit
522fi
523
524
525
526# -------------------------------------------------------------------------------------- #
527# SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP #
528# -------------------------------------------------------------------------------------- #
529# #
530# The lines below define the basic setup for the database and give examples and #
531# explanations for the various options available. #
532# The request of the data itself is done with a smaller setup further down. #
533# #
534# -------------------------------------------------------------------------------------- #
535#
536# ----------
537# DB SETUP
538# ----------
539# path to file with mysql password
540sqlpw=/home/$USER/.mysql.pw
541# host of mysql server with FACT DB
542#host=lp-fact # ISDC
543host=10.0.100.21 # LP or LP via vpn
544#host=localhost # your local machine in case you have a copy of DB
545# name of database
546dbname=factdata
547#
548# -------------
549# BASIC SETUP
550# -------------
551# output path
552path=`dirname $0`
553datapath=$path"/data"
554# create directory for data files
555if ! [ -e $datapath ]
556then
557 mkdir $datapath
558fi
559# time unit
560#timeunit=timestamp # default
561#timeunit=unix
562timeunit=mjd
563# time binning
564# positive values: minutes
565# negative values: days
566# special case 0: period
567# for season binning choose -365 and according start date
568#bin=20 # minutes
569#bin=0 # period
570bin=-1 # nightly
571#bin=-365 # yearly
572# choose analysis
573#table="AnalysisResultsAllQLA" # N/A
574table="AnalysisResultsRunLP" # QLA
575#table="AnalysisResultsRunISDC" # ISDC
576# time range
577nightmin=20111115
578nightmax=20201231
579# overwrite dataset file?
580# (useful to combine different binnings in one file -> set to "no")
581overwrite="yes"
582# optional: require minimal ontime per bin (default 20 min)
583#ontimelimit=30 # 30 min
584ontimelimit= # default 20 min
585# data quality selection
586# if you explicitely don't want a datacheck, you can comment the following line
587usedch="yes"
588# use your own datacheck instead
589# use a line like the following defining your own data quality selection cut
590#dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
591# apply additional predefined cuts
592# light conditions
593#light="nomoon" # only data with no moon (but twilight allowed)
594#light="dark" # only dark night data
595# TNG dust - cut away data with calima
596#dust=1
597#dust=10
598
599
600# -------------------------------------------------------------------------------------- #
601# SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE #
602# -------------------------------------------------------------------------------------- #
603# #
604# Adapt the lines below to your needs. #
605# Overwrite default settings above. #
606# The data-request is sent with the line 'get_results.' #
607# Minumum setup: Define source key and name for file. #
608# The list of source keys can be found at #
609# https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+ #
610# More examples can be found further down. #
611# #
612# REMARKS: #
613# - correction of effect of zd and threshold not yet finalized and only valid for QLA #
614# - no CU-conversion available for ISDC-analysis so far (that for QLA is used instead) #
615# #
616# -------------------------------------------------------------------------------------- #
617
618# 501 MAGIC
619source=2
620name="Mrk501_2014_forMAGIC"
621bin=-1
622nightmin=20140714
623nightmax=20140805
624get_results
625
626bin=30
627name="Mrk501_2014_forMAGIC30"
628get_results
629
630bin=0
631name="P"
632nightmin=20140501
633nightmax=20140930
634get_results
635
636bin=20
637nightmin=20140623
638nightmax=20140623
639name="Mrk501_test"
640get_results
641
642
643# end script here
644exit
645
646
647
648#
649# more examples
650#
651
652# Mrk 421
653source=1
654name="Mrk421_nightly"
655bin=-1
656get_results
657name="Mrk421_20min"
658bin=20
659get_results
660name="Mrk421_3d"
661bin=-3
662get_results
663name="Mrk421_10d"
664bin=-10
665get_results
666name="Mrk421_period"
667bin=0
668get_results
669
670
671
672# Mrk 501
673source=2
674name="Mrk501_nightly"
675bin=-1
676get_results
677name="Mrk501_20min"
678bin=20
679get_results
680name="Mrk501_3d"
681bin=-3
682get_results
683name="Mrk501_10d"
684bin=-10
685get_results
686name="Mrk501_period"
687bin=0
688get_results
689
690
691
692# 2344
693source=3
694name="2344_nightly"
695bin=-1
696get_results
697name="2344_20min"
698bin=20
699get_results
700name="2344_period"
701bin=0
702get_results
703
704
705
706# 1959
707source=7
708name="1959_nightly"
709bin=-1
710get_results
711name="1959_20min"
712bin=20
713get_results
714name="1959_period"
715bin=0
716get_results
717
718
719
720# 0323
721source=12
722name="0323_nightly"
723bin=-1
724get_results
725name="0323_20min"
726bin=20
727get_results
728name="0323_period"
729bin=0
730get_results
731
732
733
734# crab
735source=5
736name="Crab_nightly"
737bin=-1
738get_results
739name="Crab_20min"
740bin=20
741get_results
742name="Crab_period"
743bin=0
744get_results
745name="Crab_season"
746bin=-365
747nightmin=20110716
748nightmax=20180716
749get_results
750
751
752
753name="1959_2016"
754source=7
755bin=-1
756nightmin=20160201
757nightmax=20161105
758get_results
759
760name="1959_all_variable"
761overwrite="no"
762source=7
763bin=-365
764nightmin=20120201
765nightmax=20130131
766get_results
767nightmin=20130201
768nightmax=20140131
769get_results
770nightmin=20140201
771nightmax=20150131
772get_results
773bin=0
774nightmin=20150201
775nightmax=20160131
776get_results
777bin=-1
778nightmin=20160201
779nightmax=20170131
780get_results
781bin=0
782nightmin=20170201
783nightmax=20180131
784get_results
785
786
787
788overwrite="yes"
789name="1959_all_variable2"
790overwrite="no"
791source=7
792bin=-365
793nightmin=20120201
794nightmax=20130131
795get_results
796nightmin=20130201
797nightmax=20140131
798get_results
799nightmin=20140201
800nightmax=20150131
801get_results
802bin=0
803nightmin=20150201
804nightmax=20160131
805get_results
806bin=-1
807nightmin=20160201
808nightmax=20160817
809get_results
810bin=0
811nightmin=20160818
812nightmax=20180131
813get_results
814
815
816
817overwrite="yes"
818bin=0
819source=3
820name="2344period"
821get_results
822
823
824
825# flare night (HESS)
826name="Mrk501_10min_flarenight"
827source=2
828bin=10
829nightmin=20140623
830nightmax=20140623
831get_results
832
833
834
835# flare night (HESS)
836name="Mrk501_5min_flarenight"
837source=2
838bin=5
839nightmin=20140623
840nightmax=20140623
841get_results
842
843
844
845
846# full sample
847name="Mrk421_all_nightly"
848source=1
849get_results
850
851name="Mrk501_all_nightly"
852source=2
853get_results
854
855name="1959_all_nightly"
856source=7
857get_results
858
859name="2344_all_nightly"
860source=3
861get_results
862
863
864
865name="HESE20160427"
866source=19
867nightmin=20160425
868bin=-10
869get_results
870
871name="AMON20160731"
872source=21
873nightmin=20160730
874bin=-10
875get_results
876
877
878
Note: See TracBrowser for help on using the repository browser.