Index: /trunk/DataCheck/Tools/get_data.sh
===================================================================
--- /trunk/DataCheck/Tools/get_data.sh	(revision 19730)
+++ /trunk/DataCheck/Tools/get_data.sh	(revision 19731)
@@ -137,4 +137,8 @@
          echo "#   Calima Cut: dust < "$dust" ug/cm3"
       fi
+      if [ "$factorcut" != "" ]
+      then
+         echo "#   Factor Cut: zdfactor*thfactor > "$factorcut
+      fi
       if [ "$usedch" == "yes" ]
       then
@@ -175,5 +179,5 @@
       zdfactor="(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5))" 
       #zdfactor="(1/(pow(cos(fZenithDistanceMean*PI()/180),3)+14.8/21.9*pow(sin(2*fZenithDistanceMean*PI()/180),5)))" 
-      thfactor="(1-0.00124/1.21*(fThresholdMinSet-500)*(fThresholdMinSet>=500))"                                 
+      thfactor="(1-0.00124/1.21*(fThresholdMinSet-500)*(fThresholdMinSet>=500))"
       # ETh: 1100 GeV 
       crabflux="1.81"
@@ -207,14 +211,23 @@
    cufactor="fCU" 
    # some calculations
-   excerr="ExcErr(Sum(fNumSigEvts), SUM(fNumBgEvts))"
+   #excerr="ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))"
+   excerr="ExcErr(fNumSigEvts, fNumBgEvts)"
    CU="SUM(fNumExcEvts/"$cufactor")/SUM("$ontimeif")*3600"
-   CUerr=$excerr"/SUM("$ontimeif")*3600*SUM(fNumExcEvts/"$cufactor")/SUM(fNumExcEvts)"
+   #CUerr=$excerr"/SUM("$ontimeif")*3600*SUM(fNumExcEvts/"$cufactor")/SUM(fNumExcEvts)"
+   CUerr="SQRT(SUM(POW("$excerr"/"$cufactor",2)))/SUM("$ontimeif")*3600"
    CUcor="SUM("$correvts"/"$cufactor")/SUM("$ontimeif")*3600"
-   CUcorerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
-   excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
+   #CUcorerr=$excerr"/SUM("$ontimeif")*3600*SUM("$correvts"/"$cufactor")/SUM(fNumExcEvts)"
+   CUcorerr="SQRT(SUM(POW("$excerr"/"$cufactor"/"$zdfactor"/"$thfactor",2)))/SUM("$ontimeif")*3600"
+   #excerr2="ExcErr(SUM(o.sigevts),SUM(o.bgevts))"
+   excerr2="ExcErr(o.sigevts,o.bgevts)"
    CU2="SUM((o.sigevts-o.bgevts)/o.cufactor)/SUM(o.ot)*3600"
-   CUerr2=$excerr2"/SUM(o.ot)*3600*SUM((o.sigevts-o.bgevts)/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
-   CUcor2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
-   CUcorerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
+   #CUerr2=$excerr2"/SUM(o.ot)*3600*SUM((o.sigevts-o.bgevts)/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
+   CUerr2="SQRT(SUM(POW("$excerr2"/o.cufactor,2)))/SUM(o.ot)*3600"
+   #CUcor2="SUM(o.corevts/o.cufactor)/SUM(o.ot)*3600"
+   CUcor2="SUM(o.excevts/o.zdfactor/o.thfactor/o.cufactor)/SUM(o.ot)*3600"
+   #CUcorerr2=$excerr2"/SUM(o.ot)*3600*SUM(o.corevts/o.cufactor)/(SUM(o.sigevts)-SUM(o.bgevts))"
+   CUcorerr2="SQRT(SUM(POW("$excerr2"/o.cufactor/o.zdfactor/o.thfactor,2)))/SUM(o.ot)*3600"
+   
+   #"ROUND(SQRT(SUM(POW(ExcErr(fNumSigEvts, fNumBgEvts)/pow(cos(fZenithDistanceMean*PI()/180)*exp(1-cos(fZenithDistanceMean*PI()/180)),4.5)/(1.37-IF(ISNULL(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)*0.00118), 2)))/SUM( IF(ISNULL(fEffectiveOn),  fOnTimeAfterCuts ,  TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn ) )*3600, 1)"
    
    # columns to be selected
@@ -223,14 +236,16 @@
    excrate=" ROUND(SUM(fNumExcEvts)/SUM("$ontimeif")*3600, 1) AS excrate"
    significance="ROUND(LiMa(Sum(fNumSigEvts), SUM(fNumBgEvts)), 1) AS significance"
-   numexc="Sum(fNumExcEvts) AS numexc"
-   numsig="Sum(fNumSigEvts) AS numsig"
-   numbg="Sum(fNumBgEvts) AS numbg"
-   excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
+   numexc="SUM(fNumExcEvts) AS numexc"
+   numsig="SUM(fNumSigEvts) AS numsig"
+   numbg="SUM(fNumBgEvts) AS numbg"
+   #excrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600, 1) AS excrateerr"
+   excrateerr=" ROUND(ExcErr(SUM(fNumSigEvts), SUM(fNumBgEvts))/SUM("$ontimeif")*3600, 1) AS excrateerr"
    correxcrate=" ROUND(SUM("$correvts")/SUM("$ontimeif")*3600, 1) AS correxcrate"
-   correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
+   #correxcrateerr=" ROUND("$excerr"/SUM("$ontimeif")*3600*SUM("$correvts")/SUM(fNumExcEvts), 1) AS correxcrateerr"
+   correxcrateerr=" ROUND(SQRT(SUM(POW("$excerr"/"$zdfactor"/"$thfactor", 2)))/SUM("$ontimeif")*3600, 1) AS correxcrateerr"
    cu=" ROUND("$CU", 2) AS cu"
    cuerr=" ROUND("$CUerr", 2) AS cuerr"
    cucor=" ROUND("$CUcor", 2) AS cucor"
-   cucorerr=" ROUND("$CUcorerr", 2) AS cucorerr"
+   cucorerr=" ROUND("$CUcorerr", 2) AS 'cucorerr'"
    flux="ROUND("$CU" * "$crabflux", 2) AS flux"
    fluxerr="ROUND("$CUerr" * "$crabflux", 2) AS fluxerr"
@@ -239,16 +254,19 @@
    # for minute binning
    ontime2=" ROUND(SUM(o.ot)/60., 1) AS ontime"
-   excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
+   #excrate2=" ROUND((SUM(o.sigevts)-SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrate"
+   excrate2=" ROUND((SUM(o.excevts))/SUM(o.ot)*3600, 1) AS excrate"
    significance2=" ROUND(LiMa(SUM(o.sigevts),SUM(o.bgevts)), 1) AS significance"
-   numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
-   numsig2="Sum(o.sigevts) AS numsig"
-   numbg2="Sum(o.bgevts) AS numbg"
-   excrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600, 1) AS excrateerr"
-   correxcrate2=" ROUND(SUM(o.corevts)/SUM(o.ot)*3600, 1) AS correxcrate"
-   correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
+   #numexc2="Sum(o.sigevts-o.bgevts) AS numexc"
+   numexc2="SUM(o.excevts) AS numexc"
+   numsig2="SUM(o.sigevts) AS numsig"
+   numbg2="SUM(o.bgevts) AS numbg"
+   excrateerr2=" ROUND(ExcErr(SUM(o.sigevts),SUM(o.bgevts))/SUM(o.ot)*3600, 1) AS excrateerr"
+   correxcrate2=" ROUND(SUM(o.excevts/o.zdfactor/o.thfactor)/SUM(o.ot)*3600, 1) AS correxcrate"
+   #correxcrateerr2=" ROUND("$excerr2"/SUM(o.ot)*3600*SUM(o.corevts)/(SUM(o.sigevts)-SUM(o.bgevts)), 1) AS correxcrateerr"
+   correxcrateerr2=" ROUND(SQRT(SUM(POW("$excerr2"/o.zdfactor/o.thfactor,2)))/SUM(o.ot)*3600,1) AS correxcrateerr"
    cu2=" ROUND("$CU2", 2) AS cu"
    cuerr2=" ROUND("$CUerr2", 2) AS cuerr"
    cucor2=" ROUND("$CUcor2", 2) AS cucor"
-   cucorerr2=" ROUND("$CUcorerr2", 2) AS cucorerr"
+   cucorerr2=" ROUND("$CUcorerr2", 2) AS 'cucorerr'"
    flux2="ROUND("$CU2" * "$crabflux", "$fluxprec") AS flux"
    fluxerr2="ROUND("$CUerr2" *"$crabflux", "$fluxprec") AS fluxerr"
@@ -306,4 +324,9 @@
    # some sanity checks
    where=$where" AND fRunTypeKey=1 "
+   # remove empty rows
+   if [ "$rmemptyrows" == "yes" ]
+   then
+      where=$where" AND NOT ISNULL(fZenithDistanceMean) AND (NOT ISNULL(fThresholdMinSet) OR NOT ISNULL(fThresholdMedian)) AND NOT ISNULL(fCU) " 
+   fi
    # zd cut
    if [ "$zdmax" != "" ]
@@ -334,4 +357,8 @@
       where=$where" AND "$lightcut
    fi
+#   if [ "$factorcut" != "" ]
+#   then
+#      where=$where" AND ( ("$zdfactor") * ("$thfactor")) > "$factorcut" "
+#   fi
    querybase=$from$where
 
@@ -358,10 +385,10 @@
       if [ $bin -eq 0 ]
       then 
-         orderby=" fPeriod "
+         orderby=" ORDER BY fPeriod "
          #querystart=$querystart" fPeriod AS num, "
          queryend=" GROUP BY fPeriod "
       else
          num=" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) "
-         orderby=$num
+         orderby=" ORDER BY "$num
          #querystart=$querystart" FLOOR((Mjd(fRunStart)-Mjd("$nightmin")-0.5)/"`echo $bin | sed -e 's/-//'`".) AS num, "
          queryend=" GROUP BY "$num
@@ -370,8 +397,14 @@
       if [ "$ontimelimit" = "" ]
       then 
-         queryend=$queryend" HAVING SUM("$ontimeif")>1200 ORDER BY "$orderby
+         queryend=$queryend" HAVING SUM("$ontimeif")>1200 "
       else
-         queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
+         queryend=$queryend" HAVING SUM("$ontimeif")>"$ontimelimit
       fi
+#      if [ "$ontimelimit" = "" ]
+#      then 
+#         queryend=$queryend" HAVING abs(cucorerr) < 1 AND SUM("$ontimeif")>1200 ORDER BY "$orderby
+#      else
+#         queryend=$queryend" HAVING abs(cucorerr) < 1 AND SUM("$ontimeif")>"$ontimelimit" ORDER BY "$orderby
+#      fi
       
       # internal
@@ -385,6 +418,13 @@
       queryint=$queryint" MIN("$zenith"Min) AS zdmin, MAX("$zenith"Max) AS zdmax, "
       queryint=$queryint" MIN("$thresh") AS thmin, MAX("$thresh") AS thmax, "
-      queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG(fR750Cor), 2) AS R750cor,  ROUND(AVG(fR750Ref), 2) AS R750ref "
-      queryint=$queryint" "$querybase" "$querydch" "$queryend
+      queryint=$queryint" ROUND(AVG("$cufactor"), 1) AS cufactor, ROUND(AVG("$zdfactor"), 1) AS zdfactor, ROUND(AVG("$thfactor"), 1) AS thfactor, "
+      queryint=$queryint" ROUND(AVG(fR750Cor), 2) AS R750cor,  ROUND(AVG(fR750Ref), 2) AS R750ref "
+      if [ "$factorcut" == "" ]
+      then
+         queryint=$queryint" "$querybase" "$querydch" "$queryend" "$orderby
+      else
+#         queryint=$queryint" "$querybase" "$querydch" "$queryend" "$orderby
+         queryint=$queryint" "$querybase" "$querydch" "$queryend" AND ( zdfactor * thfactor > "$factorcut") "$orderby
+      fi
       
       # for collaborators
@@ -394,9 +434,16 @@
       querycol=$querycol" "$excrateerr", "$correxcrateerr", "$cucorerr", "$fluxcorerr", "
       querycol=$querycol" "$significance
-      querycol=$querycol" "$querybase" "$querydch" "$queryend
+      if [ "$factorcut" == "" ]
+      then
+         querycol=$querycol" "$querybase" "$querydch" "$queryend" "$orderby
+      else
+#         querycol=$querycol" "$querybase" "$querydch" "$queryend" "$orderby
+         querycol=$querycol" "$querybase" "$querydch" "$queryend" AND (AVG("$zdfactor") * AVG("$thfactor") > "$factorcut") "$orderby
+      fi
       
       # external
       # no datacheck applied for external files
-      queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend
+      # no cut on factors 
+      queryext=$querystart" "$excrate", "$deltat", "$excrateerr" "$querybase" "$queryend" "$orderby
       
    else
@@ -408,6 +455,8 @@
       querybase=" FROM (SELECT fNight, fZenithDistanceMin AS zdmin, fZenithDistanceMax AS zdmax, "$thresh" AS th, "
       querybase=$querybase" fR750Cor AS R750cor, fR750Ref AS R750ref, "$cufactor" AS cufactor, "
+      querybase=$querybase" "$zdfactor" AS zdfactor, "$thfactor" AS thfactor, "
       querybase=$querybase" @ot:="$ontimeif" AS ot, fRunStart AS start, fRunStop AS stop, "
-      querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
+      #querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, "$correvts" AS corevts, "
+      querybase=$querybase" fNumSigEvts AS sigevts, fNumBgEvts AS bgevts, fNumExcEvts AS excevts, "
       querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @bl, @bl := @bl + 1) AS block, "
       querybase=$querybase" IF (@night=fNight AND FLOOR((@os+@ot)/"$bin"./60.)<1, @os:=@os + @ot, @os := @ot) AS os, @night :=fNight AS night "
@@ -421,5 +470,6 @@
       queryint=$queryint" "$excrateerr2", "$cuerr2", "$fluxerr2", "$correxcrateerr2", "$cucorerr2", "$fluxcorerr2", "
       queryint=$queryint" "$significance2", "
-      queryint=$queryint" avg(o.night) AS night, "
+      #queryint=$queryint" avg(o.night) AS night, "
+      queryint=$queryint" min(o.night) AS nightmin, max(o.night) AS nightmax, "
       queryint=$queryint" "$numexc2", "$numsig2", "$numbg2", "
       queryint=$queryint" MIN(o.zdmin) AS zdmin, MAX(o.zdmax) AS zdmax, MIN(o.th) AS thmin, MAX(o.th) AS thmax, "
@@ -480,6 +530,6 @@
    echo "#" >> $fileint
    headerint="# time["$timeunit"] start["$timeunit"] stop["$timeunit"] excess-rate[evts/h] flux[CU] flux[e-11ph/cm2/s] corrected_excess-rate[evts/h] corrected_flux[CU] corrected_flux[e-11ph/cm2/s] (stop-start)/2["$timeunit"] ontime[min]"
-   headerint=$headerint" excess-rate_error[evts/h] flux_error[CU] flux_error[e-11ph/cm2/s] corrected_excess-rate_error[evts/h] corrected_flux_error[CU] corrected_flux_error[e-11/cm2/s] significance night num_exc num_sig num_bg "
-   headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(R750cor) avg(R750ref) "
+   headerint=$headerint" excess-rate_error[evts/h] flux_error[CU] flux_error[e-11ph/cm2/s] corrected_excess-rate_error[evts/h] corrected_flux_error[CU] corrected_flux_error[e-11/cm2/s] significance nightmin nightmax num_exc num_sig num_bg "
+   headerint=$headerint" zdmin zdmax thmin thmax avg(cufactor) avg(zdfactor) avg(thfactor) avg(R750cor) avg(R750ref) "
    echo $headerint >> $fileint
    #echo "$queryint"
@@ -639,4 +689,8 @@
 # use different conversion from CU to fluxes
 #crabfluxconv="2.5"
+# remove lines which contain NULL (e.g. zd-corrected flux when zd=NULL)
+rmemptyrows="yes"
+# remove lines with zdfactor * thfactor > value
+factorcut=0.3
 
 
@@ -655,13 +709,193 @@
 # -------------------------------------------------------------------------------------- #
 
-# some test
-table="AnalysisResultsRunCutsLC" # CutsLC
-bin=0
+
+datapath=/home/dorner/FACT.analysis/corrected.lcs.for.collaborators/lcs
+
+# put your data request here, examples below
+
+# LCs for 421 campaign with Astrosat Jan 2019
+table="AnalysisResultsRunLP" 
+# Mrk 421
+source=1
+# nightly 
+bin=-1
+nightmin=20190109
+nightmax=20190113
+name="QLA_Mrk421_nightly_Astrosat-Jan-2019"
+get_results
+# 20min
+bin=20
+name="QLA_Mrk421_20min_Astrosat-Jan-2019"
+get_results
+# 30min
+bin=30
+name="QLA_Mrk421_30min_Astrosat-Jan-2019"
+get_results
+# 40min
+bin=40
+name="QLA_Mrk421_40min_Astrosat-Jan-2019"
+get_results
+# 60min
+bin=60
+name="QLA_Mrk421_60min_Astrosat-Jan-2019"
+get_results
+# 80min
+bin=80
+name="QLA_Mrk421_80min_Astrosat-Jan-2019"
+get_results
+# 90min
+bin=90
+name="QLA_Mrk421_90min_Astrosat-Jan-2019"
+get_results
+
+
+
+
+exit
+
+# LCs for SED project 2013 
+table="AnalysisResultsRunCutsLC" # CutsLC
+# Mrk 421
+source=1
+# nightly 
+bin=-1
+nightmin=20120901
+nightmax=20130516
+name="Mrk421_nightly_SED-project-2013"
+get_results
+
+
+
+# LCs for XMM proposal
+table="AnalysisResultsRunCutsLC" # CutsLC
+# Mrk 421
+source=1
+# nightly 
+bin=-1
+nightmin=20111115
+nightmax=20201231
+name="Mrk421_nightly_XMM-proposal-2019"
+get_results
 # Mrk 501
 source=2
-name="Mrk501_all_P"
-get_results
-
-#exit
+name="Mrk501_nightly_XMM-proposal-2019"
+get_results
+# 1959
+source=7
+name="1959_nightly_XMM-proposal-2019"
+get_results
+
+
+
+# LCs for 421 campaign with Astrosat Jan 2019
+table="AnalysisResultsRunCutsLC" # CutsLC
+# Mrk 421
+source=1
+# nightly 
+bin=-1
+nightmin=20190109
+nightmax=20190113
+name="Mrk421_nightly_Astrosat-Jan-2019"
+get_results
+# 20min
+bin=20
+name="Mrk421_20min_Astrosat-Jan-2019"
+get_results
+# 30min
+bin=30
+name="Mrk421_30min_Astrosat-Jan-2019"
+get_results
+# 40min
+bin=40
+name="Mrk421_40min_Astrosat-Jan-2019"
+get_results
+# 60min
+bin=60
+name="Mrk421_60min_Astrosat-Jan-2019"
+get_results
+# 80min
+bin=80
+name="Mrk421_80min_Astrosat-Jan-2019"
+get_results
+# 90min
+bin=90
+name="Mrk421_90min_Astrosat-Jan-2019"
+get_results
+
+
+
+# LCs for flaring pattern project 
+table="AnalysisResultsRunCutsLC" # CutsLC
+# Mrk 421
+source=1
+# nightly 
+bin=-1
+nightmin=20111115
+nightmax=20201231
+name="Mrk421_nightly_Flaring-pattern-project"
+get_results
+# 2d
+bin=-2
+nightmin=20111115
+nightmax=20201231
+name="Mrk421_2day_Flaring-pattern-project"
+get_results
+
+
+
+# LCs for ToO-Trigger project 
+table="AnalysisResultsRunCutsLC" # CutsLC
+# Mrk 421
+source=1
+# nightly for 2019
+bin=-1
+nightmin=20190101
+nightmax=20190630
+name="Mrk421_nightly_20190101-20190630_ToO-project-2019"
+get_results
+# 20 min around flare
+bin=20
+nightmin=20190609
+nightmax=20190612
+name="Mrk421_20min_20190609-20190612_ToO-project-2019"
+get_results
+
+exit
+
+table="AnalysisResultsRunLP" # QLA
+# Mrk 421
+source=1
+# nightly for 2019
+bin=-1
+nightmin=20190101
+nightmax=20190630
+name="QLA_Mrk421_nightly_20190101-20190630_ToO-project-2019"
+get_results
+# 20 min around flare
+bin=20
+nightmin=20190609
+nightmax=20190612
+name="QLA_Mrk421_20min_20190609-20190612_ToO-project-2019"
+get_results
+
+
+table="AnalysisResultsRunISDC" # ISDC
+# Mrk 421
+source=1
+# nightly for 2019
+bin=-1
+nightmin=20190101
+nightmax=20190630
+name="ISDC_Mrk421_nightly_20190101-20190630_ToO-project-2019"
+get_results
+# 20 min around flare
+bin=20
+nightmin=20190609
+nightmax=20190612
+name="ISDC_Mrk421_20min_20190609-20190612_ToO-project-2019"
+get_results
+
+
+exit
 
 # LC for ICRC
@@ -692,5 +926,5 @@
 
 
-#exit
+exit
 
 # Mrk 501