Changeset 19028 for trunk/DataCheck


Ignore:
Timestamp:
06/26/18 23:34:00 (7 years ago)
Author:
Daniela Dorner
Message:
updated data check, fixed order by, fixed bug, added more comments and explanation
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Tools/get_data.sh

    r18895 r19028  
    2828   # some basic query parts
    2929
    30    # DataCheck (old)
    31    ##data with old feedback and/or different bias voltage
    32    #query=$query" AND fNight>20120420 AND NOT fNight IN (20120406,20120410,20120503) AND"
    33    ## broken bias channel
    34    #query=$query" NOT fNight BETWEEN 20121206 AND 20130110"
    35    # bg-rate cut
    36    zdparam=" pow(0.753833*cos(Radians(fZenithDistanceMean)), 7.647435)*exp(-5.753686*pow(Radians(fZenithDistanceMean),2.089609))"
    37    thparam=" pow((if(isnull(fThresholdMinSet),fThresholdMedian,fThresholdMinSet)-329.4203),2)*(-0.0000002044803) "
    38    param=" (fNumEvtsAfterBgCuts/5-fNumSigEvts)/fOnTimeAfterCuts - "$zdparam" - "$thparam" "
    39    dchold=" -0.085 < ("$param") "
    40    dchold=$dchold" AND 0.25 > ("$param") "
    41    # Datacheck (new) -> combine
    42    dchval=" fNumEvtsAfterBgCuts/(1.41*POW(fZenithDistanceMean*PI()/180,2)+0.975)/(-7.53e-12*POW(10, LOG10(fThresholdMinSet)*3.69)+1.035)/TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))/fEffectiveOn "
    43    # some semi-automatic datacheck
    44    dch="  AND (("$dchval" BETWEEN 0.8 AND 1.7 AND fNight BETWEEN 20140520 AND 20150131) " #A
    45    dch=$dch" OR  ("$dchval" BETWEEN 0.4 AND 1.6 AND fNight BETWEEN 20150201 AND 20150715) " #B
    46    dch=$dch" OR  ("$dchval" BETWEEN 0.7 AND 1.4 AND fNight BETWEEN 20150716 AND 20160218) " #C
    47    dch=$dch" OR  ("$dchval" BETWEEN 0.5 AND 1.0 AND fNight > 20160220) " #D
    48    dch=$dch" OR  ("$dchold" AND fNight<20140520)) " #old
    49 
     30   # data check based on artificial trigger rate
     31   #dch=" AND fR750Cor/fR750Ref >0.93 "
     32   dch=" AND fR750Cor/fR750Ref BETWEEN 0.93 AND 1.3 "
     33   # ontime
    5034   ontime1=" TIME_TO_SEC(TIMEDIFF(fRunStop,fRunStart))*fEffectiveOn "
    5135   ontime2=" fOnTimeAfterCuts "
    … …  
    5640   # some sanity checks
    5741   where=$where" AND fRunTypeKey=1 "
    58 #   where=$where" AND NOT ISNULL(fNumSigEvts) AND NOT ISNULL(fNumBgEvts) "
    59 #   where=$where" AND NOT fRunStart='0000-00-00 00:00:00' AND NOT fRunStop='0000-00-00 00:00:00' "
    6042   # zd cut
    6143   where=$where" AND fZenithDistanceMax < "$zdmax
    … …  
    183165      queryint=$queryint" "$time" as time, "$start" as start, "$stop" as stop, "
    184166      queryint=$queryint" round("$excrate", 1) as excrate, round("$correxcrate", 1) as correxcrate, "
    185       queryint=$queryint" round($cu, 1) as cu, $flux as flux, "
     167      queryint=$queryint" round("$cu", 2) as cu, "$flux" as flux, "
    186168      queryint=$queryint" "$deltat" as deltat, round("$ontime", 1) as ontime, "
    187169      queryint=$queryint" round("$excrateerr", 1) as excrateerr, round("$correxcrateerr", 1) as correxcrateerr, "
    188       queryint=$queryint" round($cuerr, 1) as cuerr, $fluxerr as fluxerr, "
     170      queryint=$queryint" round("$cuerr", 2) as cuerr, "$fluxerr" as fluxerr, "
    189171      queryint=$queryint" round("$significance", 1) as significance, "
    190172      queryint=$queryint" Min(fNight) as nightmin, "
    … …  
    257239      # AND NOT SUM(fNumBgEvts)=SUM(fNumSigEvts)
    258240     
    259       queryint=$queryint" "$querybase
    260       queryext=$queryext" "$querybase
     241      queryint=$queryint" "$querybase" order by 'time'"
     242      queryext=$queryext" "$querybase" order by 'time'"
    261243   fi
    262244   
    263245   
    264    fileint=$path"/data/FACT_preliminary_"$name"_internal.dat"
     246   fileint=$datapath"/FACT_preliminary_"$name"_internal.dat"
    265247   if [ "$overwrite" = "yes" ]
    266248   then
    … …  
    279261   
    280262   
    281    fileext=$path"/data/FACT_preliminary_"$name".dat"
     263   fileext=$datapath"/FACT_preliminary_"$name".dat"
    282264   if [ "$overwrite" = "yes" ]
    283265   then
    … …  
    298280# setup
    299281# db
    300 sqlpw=/home/$USER/.mysql.pw
    301 dbname=factdata20170804
    302 # selection
     282sqlpw=/home/$USER/.mysql.pw # file with mysql credentials
     283dbname=factdata # name of database
     284# defaults for zd and threshold
     285zdmax=90 # all data
     286thmax=1500 # all data
     287# output path
     288path=`dirname $0`
     289datapath=$path"/data"
     290if ! [ -e $datapath ]
     291then
     292   mkdir $datapath
     293fi
     294# time unit
     295#timeunit=timestamp # default
     296#timeunit=unix
    303297timeunit=mjd
    304 #bin=20 # min
     298# time binning
     299# positive values: minutes
     300# negative values: days
     301# special case 0: period
     302# for season binning choose -365 and according start date
     303#bin=20 # minutes
    305304#bin=0 # period
     305bin=-1 # nightly
    306306#bin=-365 # yearly
    307 bin=-1 # nightly
    308 zdmax=90
    309 thmax=1500
    310 path=`dirname $0`
    311 table="AnalysisResultsRunLP"
    312 table="AnalysisResultsAllQLA"
    313 overwrite="yes"
    314 bin=-1 # nightly
     307# choose analysis
     308#table="AnalysisResultsAllQLA" # N/A
     309#table="AnalysisResultsRunLP" # QLA
     310table="AnalysisResultsRunISDC"  # ISDC
     311# time range
    315312nightmin=20111115
    316313nightmax=20171231
    317 
    318 
    319 # Mrk 421
    320 source=1
    321 name="Mrk421_nightly"
    322 bin=-1
    323 get_results
    324 name="Mrk421_20min"
    325 bin=20
    326 get_results
    327 name="Mrk421_3d"
    328 bin=-3
    329 get_results
    330 name="Mrk421_10d"
    331 bin=-10
    332 get_results
    333 name="Mrk421_period"
    334 bin=0
    335 get_results
    336 
    337 
    338 
    339 # Mrk 501
    340 source=2
    341 name="Mrk501_nightly"
    342 bin=-1
    343 get_results
    344 name="Mrk501_20min"
    345 bin=20
    346 get_results
    347 name="Mrk501_3d"
    348 bin=-3
    349 get_results
    350 name="Mrk501_10d"
    351 bin=-10
    352 get_results
    353 name="Mrk501_period"
    354 bin=0
    355 get_results
    356 
    357 
    358 
    359 # 2344
    360 source=3
    361 name="2344_nightly"
    362 bin=-1
    363 get_results
    364 name="2344_20min"
    365 bin=20
    366 get_results
    367 name="2344_period"
    368 bin=0
    369 get_results
    370 
    371 
    372 
    373 # 1959
    374 source=7
    375 name="1959_nightly"
    376 bin=-1
    377 get_results
    378 name="1959_20min"
    379 bin=20
    380 get_results
    381 name="1959_period"
    382 bin=0
    383 get_results
    384 
    385 
    386 
    387 # 0323
    388 source=12
    389 name="0323_nightly"
    390 bin=-1
    391 get_results
    392 name="0323_20min"
    393 bin=20
    394 get_results
    395 name="0323_period"
    396 bin=0
    397 get_results
    398 
    399 
     314# overwrite dataset file?
     315# (useful to combine different binnings in one file -> set to "no")
     316overwrite="yes"
     317
     318
     319# example (adapt to your needs)
    400320
    401321# crab
    … …  
    418338
    419339
     340# end script here
    420341exit
    421342
     343
     344
     345#
    422346# more examples
     347#
     348
     349# Mrk 421
     350source=1
     351name="Mrk421_nightly"
     352bin=-1
     353get_results
     354name="Mrk421_20min"
     355bin=20
     356get_results
     357name="Mrk421_3d"
     358bin=-3
     359get_results
     360name="Mrk421_10d"
     361bin=-10
     362get_results
     363name="Mrk421_period"
     364bin=0
     365get_results
     366
     367
     368
     369# Mrk 501
     370source=2
     371name="Mrk501_nightly"
     372bin=-1
     373get_results
     374name="Mrk501_20min"
     375bin=20
     376get_results
     377name="Mrk501_3d"
     378bin=-3
     379get_results
     380name="Mrk501_10d"
     381bin=-10
     382get_results
     383name="Mrk501_period"
     384bin=0
     385get_results
     386
     387
     388
     389# 2344
     390source=3
     391name="2344_nightly"
     392bin=-1
     393get_results
     394name="2344_20min"
     395bin=20
     396get_results
     397name="2344_period"
     398bin=0
     399get_results
     400
     401
     402
     403# 1959
     404source=7
     405name="1959_nightly"
     406bin=-1
     407get_results
     408name="1959_20min"
     409bin=20
     410get_results
     411name="1959_period"
     412bin=0
     413get_results
     414
     415
     416
     417# 0323
     418source=12
     419name="0323_nightly"
     420bin=-1
     421get_results
     422name="0323_20min"
     423bin=20
     424get_results
     425name="0323_period"
     426bin=0
     427get_results
     428
     429
     430
     431# crab
     432source=5
     433name="Crab_nightly"
     434bin=-1
     435get_results
     436name="Crab_20min"
     437bin=20
     438get_results
     439name="Crab_period"
     440bin=0
     441get_results
     442name="Crab_season"
     443bin=-365
     444nightmin=20110716
     445nightmax=20180716
     446get_results
     447
     448
    423449
    424450name="1959_2016"
    … …  
    486512
    487513
    488 exit
    489 
    490514overwrite="yes"
    491 
    492515bin=0
    493516source=3
    … …  
    495518get_results
    496519
    497 exit
     520
    498521
    499522# flare night (HESS)
    … …  
    506529
    507530
    508 exit
    509531
    510532# flare night (HESS)
    … …  
    517539
    518540
    519 exit
    520541
    521542
    … …  
    537558get_results
    538559
    539 exit
    540560
    541561
    … …  
    553573
    554574
    555 # full sample
    556 name="Mrk501_all_3d"
    557 source=2
    558 bin=-3
    559 get_results
    560 
    561 exit
    562 
    563 # full sample
    564 name="Mrk421_all_3d"
    565 source=1
    566 bin=-3
    567 get_results
    568 
    569 name="1959_2016_nightly"
    570 source=7
    571 nightmin=20160301
    572 nightmax=20160831
    573 get_results
    574 
    575 # full sample
    576 name="Mrk421_all_nightly"
    577 source=1
    578 get_results
    579 
    580 name="Mrk501_all_nightly"
    581 source=2
    582 get_results
    583 
    584 name="1959_all_nightly"
    585 source=7
    586 get_results
    587 
    588 bin=20 # min
    589 
    590 # full sample
    591 name="Mrk421_all_20min"
    592 source=1
    593 get_results
    594 
    595 name="Mrk501_all_20min"
    596 source=2
    597 get_results
    598 
    599 name="1959_all_20min"
    600 source=7
    601 get_results
    602 
    603 exit
    604 
    605 name="Mrk501_all_weekly"
    606 nightmin=20140524
    607 nightmax=20140930
    608 source=2
    609 bin=-7
    610 get_results
    611 
    612 exit
    613 
    614 
     575
Note: See TracChangeset for help on using the changeset viewer.