Changeset 19033 for trunk/DataCheck


Ignore:
Timestamp:
06/28/18 15:49:59 (6 years ago)
Author:
Daniela Dorner
Message:
added tutorial, added comments, restructured query functions, added file for collaborators, reworked columns for 3 files, added zd and th for internal
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Tools/get_data.sh

    r19032 r19033  
    11#!/bin/bash
    22
    3 # ----------------------------------------------------------------
    4 #  README README README README README README README README README
    5 # ----------------------------------------------------------------
    6 #                                                                #
    7 # To use this script, you need                                   #
    8 #   - a computer with access to the FACT database in La Palma    #
    9 #   - a file with the password of a valid mysql-user             #
    10 #   - to define the setup below for                              #
    11 #     a) the DB access                                           #
    12 #     b) the data you want to have                               #
    13 #                                                                #
    14 # To define the setup, search for SETUP in this script and       #
    15 #   read the details there (starting roughly line 295)           #
    16 #                                                                #
    17 # Per data request, you get up to 3 files:                       #
    18 #   *_internal.dat                                               #
    19 #   *_collaborators.dat                                          #
    20 #   *_external.dat (only if binning is 20min or nightly)         #
    21 #                                                                #
    22 # Please have in mind that this started as a tool for myself,   #
    23 #  then others started using it. Also the script is not yet      #
    24 #  finalized. In case you find problems and/or have a feature    #
    25 #  request, please send and email to dorner@astro.uni-wuerzburg.de                                 #
    26 #                                                                #
    27 # ----------------------------------------------------------------
    28 #  README README README README README README README README README
    29 # ----------------------------------------------------------------
     3# ---------------------------------------------------------------- #
     4#  README README README README README README README README README  #
     5# ---------------------------------------------------------------- #
     6#                                                                  #
     7# To use this script, you need                                     #
     8#   - a computer with access to the FACT database in La Palma      #
     9#   - a file with the password of a valid mysql-user               #
     10#   - to define the setup below for                                #
     11#     a) the DB access                                             #
     12#     b) the data you want to have                                 #
     13#                                                                  #
     14# To define the setup, search for SETUP in this script and         #
     15#   read the details there                                         #
     16#                                                                  #
     17# Per data request, you get up to 3 files:                         #
     18#   *_internal.dat                                                 #
     19#   *_collaborators.dat                                            #
     20#   *_external.dat (only if binning is 20min or nightly)           #
     21#                                                                  #
     22# Please have in mind that this started as a tool for myself, then #
     23# others started using it. Also the script is not yet finalized.   #
     24# In case you find problems and/or have a feature request, please  #
     25# send and email to dorner@astro.uni-wuerzburg.de                  #
     26#                                                                  #
     27# ---------------------------------------------------------------- #
     28#  README README README README README README README README README  #
     29# ---------------------------------------------------------------- #
    3030
    3131
    … …  
    3535# ToDo (notes DD):
    3636# ----------------
    37 # - add file for collaborators
    38 # - update columns and content for 3 types of files
    39 # - limit creation of file for externals to
     37# - limit creation of file for externals to nightly and 20 min
     38# - add zd, th for internal for minute binning (daily done)
    4039# - update function for correction
    4140# - update CU for QLA
    4241# - add CU for ISDC analysis
    43 # - add zd, th for internal
    4442# - add < 20121212 data for QLA
    4543# - check crab flux
    … …  
    111109   
    112110   case $timeunit in
    113       mjd)  start=" Mjd(Min(fRunStart)) "
    114             stop=" Mjd(MAX(fRunStop)) "
    115             deltat=" (Mjd(MAX(fRunStop))-Mjd(Min(fRunStart)))/2 "
    116             time=" Mjd(Min(fRunStart))+"$deltat
    117             start2=" Mjd(MIN(o.start)) "
    118             stop2=" Mjd(MAX(o.stop)) "
    119             deltat2=" (Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2 "
    120             time2=" Mjd(MIN(o.start))+"$deltat2
     111      mjd)  delta="(Mjd(MAX(fRunStop))-Mjd(Min(fRunStart)))/2"
     112            start=" Mjd(Min(fRunStart)) AS start"
     113            stop=" Mjd(MAX(fRunStop)) AS stop"
     114            deltat=$delta" AS deltat"
     115            time=" Mjd(Min(fRunStart))+"$delta" AS time"
     116            delta2="(Mjd(MAX(o.stop))-Mjd(MIN(o.start)))/2"
     117            start2=" Mjd(MIN(o.start)) AS start"
     118            stop2=" Mjd(MAX(o.stop)) AS stop"
     119            deltat2=$delta2" AS deltat"
     120            time2=" Mjd(MIN(o.start))+"$delta2" AS time"
    121121            ;;
    122       unix) start="Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')) "
    123             stop="Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) "
    124             deltat=" (Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')))/2 "
    125             time="  Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM'))+"$deltat
    126             startstop2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start, "
    127             startstop2=$starstop2" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop, "
    128             time2="  (Unix_timestamp(CONVERT_TZ(Max(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(Min(o.start), '+00:00', 'SYSTEM')))/2 "
    129             time2="  Unix_timestamp(CONVERT_TZ(Min(o.start), '+00:00', 'SYSTEM'))+"$deltat2
     122      unix) delta="(Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')))/2"
     123            start="Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM')) AS start"
     124            stop="Unix_timestamp(CONVERT_TZ(Max(fRunStop), '+00:00', 'SYSTEM')) AS stop"
     125            deltat=$delta" AS deltat"
     126            time=" Unix_timestamp(CONVERT_TZ(Min(fRunStart), '+00:00', 'SYSTEM'))+"$delta" AS time"
     127            delta2="(Unix_timestamp(CONVERT_TZ(Max(o.stop), '+00:00', 'SYSTEM')) - Unix_timestamp(CONVERT_TZ(Min(o.start), '+00:00', 'SYSTEM')))/2"
     128            startstop2=" Unix_timestamp(CONVERT_TZ(MIN(o.start), '+00:00', 'SYSTEM')) AS start"
     129            startstop2=$starstop2" Unix_timestamp(CONVERT_TZ(MAX(o.stop), '+00:00', 'SYSTEM')) AS stop"
     130            deltat2=$delta2" AS deltat"
     131            time2=" Unix_timestamp(CONVERT_TZ(Min(o.start), '+00:00', 'SYSTEM'))+"$delta2" AS time"
    130132            ;;
    131       *)    start=" MIN(fRunStart) "
    132             stop=" MAX(fRunStop) "
    133             deltat=" sec_to_time(time_to_sec(timediff(MAX(fRunStop), Min(fRunStart)))/2) "
    134             time=" addtime(Min(fRunStart), "$deltat") "
    135             start2=" MIN(o.start) "
    136             stop2=" MAX(o.stop) "
    137             deltat2=" sec_to_time(time_to_sec(timediff(MAX(o.stop), Min(o.start)))/2) "
    138             time2=" addtime(Min(o.start), "$deltat2") "
     133      *)    delta="sec_to_time(time_to_sec(timediff(MAX(fRunStop), Min(fRunStart)))/2)"
     134            start=" MIN(fRunStart) AS start"
     135            stop=" MAX(fRunStop) AS stop"
     136            deltat=$delta" AS deltat"
     137            time=" addtime(Min(fRunStart), "$delta") AS time"
     138            delta2="sec_to_time(time_to_sec(timediff(MAX(o.stop), Min(o.start)))/2)"
     139            start2=" MIN(o.start) AS start"
     140            stop2=" MAX(o.stop) AS stop"
     141            deltat2=$delta" AS deltat"
     142            time2=" addtime(Min(o.start), "$delta2") AS time"
    139143            ;;
    140144   esac
    141    ontime=" SUM("$ontimeif")/60."
    142    ontime2=" SUM(o.ot)/60. "
    143    
    144    excrate=" SUM(fNumExcEvts)/SUM("$ontimeif")*3600 "
     145   zenith="fZenithDistance"
     146   thresh="IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)"
     147   # thomas correction factor
     148   correvts=" fNumExcEvts*(pow(cos("$zenith"Mean*PI()/180),3)+14.8/21.9*pow(sin(2*"$zenith"Mean*PI()/180),5))/((1-0.00124/1.21*("$thresh"-500)*("$thresh">=500))) "
    145149   excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
    146    significance="LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts))"
    147    numexc="Sum(fNumExcEvts)"
    148    numsig="Sum(fNumSigEvts)"
    149    numbg="Sum(fNumBgEvts)"
    150    excrateerr=" "$excerr"/SUM("$ontimeif")*3600 "
    151    # thomas correction factor
    152    correvts=" fNumExcEvts*(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))/((1-0.00124/1.21*(if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)-500)*(if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)>=500))) "
    153    correxcrate=" SUM("$correvts")/SUM("$ontimeif")*3600 "
     150   CU="SUM("$correvts"/CUQLA(fNight))/SUM("$ontimeif")*3600"
     151   CUerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/CUQLA(fNight))/SUM(fNumExcEvts)"
     152   excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
     153   CU2="SUM(o.corevts/o.cu)/SUM(o.ot)*3600"
     154   CUerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cu)/(SUM(o.sigevts)-SUM(o.bgevts))"
     155   
     156   ontime=" ROUND(SUM("$ontimeif")/60., 1) AS ontime"
     157   excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
     158   significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
     159   numexc="Sum(fNumExcEvts) AS numexc"
     160   numsig="Sum(fNumSigEvts) AS numsig"
     161   numbg="Sum(fNumBgEvts) AS numbg"
     162   excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
     163   correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
    154164   # corerr = MMath::ErrorExc(excevtssum+bgevtssum, bgevtssum*5, 0.2)/ontimesum*3600.*corrate/excrate;
    155    correxcrateerr=" "$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts) "
    156    # correction on run basis (not relevant for hess)
    157    #cu=$correxcrate"/"$cufactor
    158    cu=" SUM("$correvts"/CUQLA(fNight))/SUM("$ontimeif")*3600 "
    159    #cuerr=$correxcrateerr"/"$cufactor
    160    cuerr=" "$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/CUQLA(fNight))/SUM(fNumExcEvts) "
    161    flux=$cu" * "$crabflux
    162    fluxerr=$cuerr" * "$crabflux
    163 
    164    
    165    excrate2=" (SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600 "
    166    excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
    167    significance2="LiMa(SUM(o.sigevts),SUM(o.bgevts))"
    168    numexc2="Sum(o.sigevts-o.bgevts)"
    169    numsig2="Sum(o.sigevts)"
    170    numbg2="Sum(o.bgevts)"
    171    excrateerr2=" "$excerr2"/SUM(o.ot)*3600 "
    172    correxcrate2=" SUM(o.corevts)/SUM(o.ot)*3600 "
    173    correxcrateerr2=" "$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)) "
    174    #cu2=$correxcrate2"/"$cufactor
    175    cu2=" SUM(o.corevts/o.cu)/SUM(o.ot)*3600 "
    176    #cuerr2=$correxcrateerr2"/"$cufactor
    177    cuerr2=" "$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cu)/(SUM(o.sigevts)-SUM(o.bgevts)) "
    178    flux2="$cu2*"$crabflux
    179    fluxerr2="$cuerr2*"$crabflux
     165   correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
     166   # correction on run basis
     167   cu=" ROUND("$CU", 2) AS cu"
     168   cuerr=" ROUND("$CUerr", 2) AS cuerr"
     169   flux="ROUND("$CU" * "$crabflux", 2) AS flux"
     170   fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
     171   
     172   ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
     173   excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
     174   significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
     175   numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
     176   numsig2="Sum(o.sigevts) AS numsig"
     177   numbg2="Sum(o.bgevts) AS numbg"
     178   excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
     179   correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
     180   correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
     181   cu2=" ROUND("$CU2", 2) AS cu"
     182   cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
     183   flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
     184   fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
    180185
    181186   
    182187   if [ $bin -le 0 ]
    183188   then
    184       queryint="SELECT "
     189      num="#bin"
     190      # first part of the query
     191      querystart="SELECT "
    185192      if [ $bin -eq 0 ]
    186193      then
    187          queryint=$queryint" fPeriod as num, "
     194         querystart=$querystart" fPeriod AS num, "
    188195      else
    189          queryint=$queryint" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) as num, "
     196         querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
    190197      fi
    191       queryint=$queryint" "$time" as time, "$start" as start, "$stop" as stop, "
    192       queryint=$queryint" round("$excrate", 1) as excrate, round("$correxcrate", 1) as correxcrate, "
    193       queryint=$queryint" round("$cu", 2) as cu, "$flux" as flux, "
    194       queryint=$queryint" "$deltat" as deltat, round("$ontime", 1) as ontime, "
    195       queryint=$queryint" round("$excrateerr", 1) as excrateerr, round("$correxcrateerr", 1) as correxcrateerr, "
    196       queryint=$queryint" round("$cuerr", 2) as cuerr, "$fluxerr" as fluxerr, "
    197       queryint=$queryint" round("$significance", 1) as significance, "
    198       queryint=$queryint" Min(fNight) as nightmin, "
    199       queryint=$queryint" Max(fNight) as nightmax, "
    200       queryint=$queryint" "$numexc" as numexc, "
    201       queryint=$queryint" "$numsig" as numsig, "
    202       queryint=$queryint" "$numbg" as numbg "
    203      
    204       queryext="SELECT "
    205       if [ $bin -eq 0 ]
    206       then
    207          queryext=$queryext" fPeriod as num, "
    208       else
    209          queryext=$queryext" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) as num, "
    210       fi
    211       queryext=$queryext" "$time" as time, "$start" as start, "$stop" as stop, "
    212       queryext=$queryext" round("$correxcrate", 1) as correxcrate, round("$flux", "$fluxprec") as flux, "
    213       queryext=$queryext" "$deltat" as deltat, round("$ontime", 1) as ontime, "
    214       queryext=$queryext" round("$correxcrateerr", 1) as correxcrateerr, round("$fluxerr", "$fluxprec") as fluxerr, "
    215       queryext=$queryext" round("$significance", 1) as significance "
    216      
     198      querystart=$querystart" "$time", "$start", "$stop", "
     199     
     200      # final part of the query
    217201      querybase=$from$where
    218202      querybase=$querybase" GROUP BY num "
    … …  
    224208      fi
    225209     
     210      # internal
     211      queryint=$querystart
     212      queryint=$queryint" "$excrate", "$correxcrate", "$cu", "$flux", "
     213      queryint=$queryint" "$deltat", "$ontime", "
     214      queryint=$queryint" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
     215      queryint=$queryint" "$significance", "
     216      queryint=$queryint" Min(fNight) AS nightmin, Max(fNight) AS nightmax, "
     217      queryint=$queryint" "$numexc", "$numsig", "$numbg", "
     218      queryint=$queryint" Min("$zenith"Min) AS zdmin, Max("$zenith"Max) AS zdmax, "
     219      queryint=$queryint" Min("$thresh") AS thmin, Max("$thresh") AS thmax "
    226220      queryint=$queryint" "$querybase
    227       queryext=$queryext" "$querybase
     221     
     222      # for collaborators
     223      querycol=$querystart
     224      querycol=$querycol" "$excrate", "$correxcrate", "$cu", "$flux", "
     225      querycol=$querycol" "$deltat", "$ontime", "
     226      querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cuerr", "$fluxerr", "
     227      querycol=$querycol" "$significance
     228      querycol=$querycol" "$querybase
     229     
     230      # external
     231      queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase
     232     
    228233   else
    229       queryint="SELECT "
    230       queryint=$queryint" "$time2" as time, "$start2" as start, "$stop2" as stop, "
    231       queryint=$queryint" round("$excrate2", 1) as excrate, round("$correxcrate2", 1) as correxcrate, "
    232       queryint=$queryint" round("$cu2", 1) as cu, round("$flux2", "$fluxprec") as flux, "
    233       queryint=$queryint" round("$excrateerr2", 1) as excrateerr, round("$correxcrateerr2", 1) as correxcrateerr, "
    234       queryint=$queryint" "$deltat2" as deltat, round("$ontime2", 1) as ontime, "
    235       queryint=$queryint" round("$cuerr2", 1) as cuerr, round("$fluxerr2", "$fluxprec") as fluxerr, "
    236       queryint=$queryint" round("$significance2", 1) as significance, "
    237       queryint=$queryint" avg(o.night) as night, "
    238       queryint=$queryint" "$numexc2" as numexc, "
    239       queryint=$queryint" "$numsig2" as numsig, "
    240       queryint=$queryint" "$numbg2" as numbg "
    241      
    242       queryext="SELECT "
    243       queryext=$queryext" "$time2" as time, "$start2" as start, "$stop2" as stop, "
    244       queryext=$queryext" round("$correxcrate2", 1) as correxcrate, round("$flux2", "$fluxprec") as flux, "
    245       queryext=$queryext" "$deltat2" as deltat, round("$ontime2", 1) as ontime, "
    246       queryext=$queryext" round("$correxcrateerr2", 1) as correxcrateerr, round("$fluxerr2", "$fluxprec") as fluxerr, "
    247       queryext=$queryext" round("$significance2", 1) as significance "
    248      
     234      num=
     235      # first part of the query
     236      querystart="SELECT "
     237      querystart=$querystart" "$time2", "$start2", "$stop2", "
     238     
     239      # final part of the query
    249240      querybase=" FROM (SELECT fNight, @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "
    250241      querybase=$querybase" "$correvts" AS corevts, CUQLA(fNight) AS cu, "
    … …  
    252243      querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
    253244      querybase=$querybase$from" CROSS JOIN (SELECT @night :=0, @ot :=0, @os :=0, @bl:=0) PARAMS "
    254       querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin
    255      
    256       queryint=$queryint" "$querybase" order by 'time'"
    257       queryext=$queryext" "$querybase" order by 'time'"
     245      querybase=$querybase$where" ORDER BY fRunStart) o GROUP BY block HAVING ontime>0.75*"$bin" ORDER BY 'time'"
     246     
     247      # internal
     248      queryint=$querystart
     249      queryint=$queryint" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
     250      queryint=$queryint" "$deltat2", "$ontime2", "
     251      queryint=$queryint" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
     252      queryint=$queryint" "$significance2", "
     253      queryint=$queryint" avg(o.night) AS night, "
     254      queryint=$queryint" "$numexc2", "$numsig2", "$numbg2" "
     255      queryint=$queryint" "$querybase
     256     
     257      # for collaborators
     258      querycol=$querystart
     259      querycol=$querycol" "$excrate2", "$correxcrate2", "$cu2", "$flux2", "
     260      querycol=$querycol" "$deltat2", "$ontime2", "
     261      querycol=$querycol" "$excrateerr2", "$correxcrateerr2", "$cuerr2", "$fluxerr2", "
     262      querycol=$querycol" "$significance2
     263      querycol=$querycol" "$querybase
     264     
     265      # external
     266      queryext=$querystart" "$excrate2", "$deltat2", "$ontime2", "$excrateerr2" "$querybase
     267     
    258268   fi
    259269   
    … …  
    263273   then
    264274      echo "internal: "$fileint
    265       echo "# this file was created at "`date` > $fileint
     275      echo "# This file was created at "`date` > $fileint
     276      # add query and policy here
    266277   fi
    267    if [ $bin -le 0 ]
    268    then
    269       echo "# numbin time[mjd] start[mjd] stop[mjd] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time[mjd] ontime[min] excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance nightmin, nightmax num_exc num_sig num_bg " >> $fileint
    270    else
    271       echo "# time[mjd] start[mjd] stop[mjd] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time[mjd] ontime[min] excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance nightmin, nightmax num_exc num_sig num_bg " >> $fileint
    272    fi
     278   headerint="# "$num" time["$timeunit"] start["$timeunit"] stop["$timeunit"] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time["$timeunit"] ontime[min]"
     279   headerint=$headerint" excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance night num_exc num_sig num_bg zdmin zdmax thmin thmax"
     280   echo $headerint >> $fileint
    273281   #echo "$queryint"
    274282   mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryint" >> $fileint
    … …  
    276284   
    277285   
    278    fileext=$datapath"/FACT_preliminary_"$name".dat"
     286   fileext=$datapath"/FACT_preliminary_"$name"_external.dat"
    279287   if [ "$overwrite" = "yes" ]
    280288   then
    281289      echo "external: "$fileext
    282       echo "# this file was created at "`date` > $fileext
     290      echo "# This file was created at "`date` > $fileext
     291      # add query and policy here
    283292   fi
    284    if [ $bin -lt 0 ]
    285    then
    286       echo "# numbin time[mjd] start[mjd] stop[mjd] corr.excrate[evts/h] flux[e-11/cm2/s] delta_time[mjd] ontime[min] corr.excrate_err[evts/h] flux_err[e-11/cm2/s] significance " >> $fileext
    287    else
    288       echo "# time[mjd] start[mjd] stop[mjd] corr.excrate[evts/h] flux[e-11/cm2/s] delta_time[mjd] ontime[min] corr.excrate_err[evts/h] flux_err[e-11/cm2/s] significance " >> $fileext
    289    fi
     293   headerext="# "$num" time["$timeunit"] start["$timeunit"] stop["$timeunit"] excrate[evts/h] delta_time["$timeunit"] excrate_err[evts/h] "
     294   echo $headerext >> $fileext
    290295   #echo "$queryext"
    291296   mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$queryext" >> $fileext
    292297   #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$queryext"
     298
     299   filecol=$datapath"/FACT_preliminary_"$name"_collaborators.dat"
     300   if [ "$overwrite" = "yes" ]
     301   then
     302      echo "collaborators: "$filecol
     303      echo "# This file was created at "`date` > $filecol
     304      # add query and policy here
     305   fi
     306   headercol="# "$num" time["$timeunit"] start["$timeunit"] stop["$timeunit"] excrate[evts/h] corr.excrate[evts/h] flux[CU] flux[e-11/cm2/s] delta_time["$timeunit"] ontime[min]"
     307   headercol=$headercol" excrate_err[evts/h] corr.excrate_err[evts/h] flux_err[CU] flux_err[e-11/cm2/s] significance "
     308   echo $headercol >> $filecol
     309   #echo "$querycol"
     310   mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -s -e "$querycol" >> $filecol
     311   #mysql --defaults-file=$sqlpw -u factread --host=$host $dbname -e "$querycol
    293312}
    294313
    295 # SETUP:
    296 # ------
    297 # DB SETUP:
    298 # ---------
     314# -------------------------------------------------------------------------------------- #
     315#  SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP SETUP   #
     316# -------------------------------------------------------------------------------------- #
     317#                                                                                        #
     318# The lines below define the basic setup for the database and give examples and          #
     319# explanations for the various options available.                                        #
     320# The request of the data itself is done with a smaller setup further down.              #
     321#                                                                                        #
     322# -------------------------------------------------------------------------------------- #
     323#
     324# ----------
     325#  DB SETUP
     326# ----------
    299327# path to file with mysql password
    300328sqlpw=/home/$USER/.mysql.pw
    … …  
    305333# name of database
    306334dbname=factdata
    307 # defaults for zd and threshold
    308 zdmax=90 # all data
    309 thmax=1500 # all data
    310335#
    311 # SETUP for your data:
    312 # --------------------
     336# -------------
     337#  BASIC SETUP
     338# -------------
    313339# output path
    314340path=`dirname $0`
    … …  
    338364# time range
    339365nightmin=20111115
    340 nightmax=20171231
     366nightmax=20201231
     367# defaults for zd and threshold
     368zdmax=90 # all data
     369thmax=1500 # all data
    341370# overwrite dataset file?
    342371# (useful to combine different binnings in one file -> set to "no")
    343372overwrite="yes"
    344 
    345 
    346 # example (adapt to your needs)
     373# optional: require minimal ontime per bin (default 20 min)
     374#ontimelimit=30 # 30 min
     375ontimelimit= # default 20 min
     376
     377# -------------------------------------------------------------------------------------- #
     378#  SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE - SETUP - GET YOUR DATA HERE  #
     379# -------------------------------------------------------------------------------------- #
     380#                                                                                        #
     381# Adapt the lines below to your needs.                                                   #
     382# Overwrite default settings above.                                                      #
     383# The  data-request is sent with the line 'get_results.'                                 #
     384# Minumum setup: Define source key and name for file.                                    #
     385# The list of source keys can be found at                                                #
     386#   https://fact-project.org/run_db/db/printtable.php?fTable=Source&fSortBy=fSourceKEY+  #
     387# More examples can be found further down.                                               #
     388#                                                                                        #
     389# REMARKS:                                                                               #
     390# - correction of effect of zd and threshold not yet finalized and only valid for QLA    #
     391# - no CU-conversion available for ISDC-analysis so far (that for QLA is used instead)   #
     392#                                                                                        #
     393# -------------------------------------------------------------------------------------- #
    347394
    348395# 501 MAGIC
    349396source=2
    350 name="Mrk501_2014_QLA"
    351 bin=-1
    352 nightmin=20140501
    353 nightmax=20140930
    354 get_results
    355 table="AnalysisResultsRunISDC"  # ISDC
    356 name="Mrk501_2014_ISDC"
    357 get_results
     397name="Mrk501_2014_forMAGIC"
     398bin=-1
     399nightmin=20140714
     400nightmax=20140805
     401get_results
     402#bin=20
     403#name="Mrk501_2014_forMAGIC20"
     404#get_results
    358405
    359406
Note: See TracChangeset for help on using the changeset viewer.