Changeset 18708


Ignore:
Timestamp:
01/05/17 00:44:59 (8 years ago)
Author:
Daniela Dorner
Message:
avoid warnings for missing files for the first few days; fixed bug in finding start and stop time; improved logging
Location:
trunk/DataCheck
Files:
15 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Processing/CheckAuxFilesAvail.sh

    r14827 r18708  
    66program=CheckAuxFilesAvail
    77step=AuxFilesAvailISDC
    8 transferdelay=3 #days
    98
    109set -C
    … …  
    2120do
    2221   night=${primaries[$s]}
    23    currentnight=`date +%Y%m%d`
    24    #checknight=`echo " $currentnight - $transferdelay " | bc -l`
    25    checknight=`date +%Y%m%d --date="-${transferdelay}day"`
    26    
    2722   setstatus "start"
    2823

  • trunk/DataCheck/Processing/CheckDriveFileAvail.sh

    r14830 r18708  
    2121do
    2222   night=${primaries[$s]}
    23    currentnight=`date +%Y%m%d`
    24    #checknight=`echo " $currentnight - $transferdelay " | bc -l`
    25    checknight=`date +%Y%m%d --date="-${transferdelay}day"`
    26    
    2723   setstatus "start"
    2824

  • trunk/DataCheck/Processing/CheckRatesFileAvail.sh

    r15384 r18708  
    2121do
    2222   night=${primaries[$s]}
    23    currentnight=`date +%Y%m%d`
    24    #checknight=`echo " $currentnight - $transferdelay " | bc -l`
    25    checknight=`date +%Y%m%d --date="-${transferdelay}day"`
    26    
    2723   setstatus "start"
    2824

  • trunk/DataCheck/Processing/CheckRawFilesAvail.sh

    r18699 r18708  
    66program=CheckRawFilesAvail
    77step=RawFileAvailISDC
    8 transferdelay=3 #days
    98failsdir=/archive/fact/fails/raw
    109
    … …  
    2322   night=${primaries[$s+$s]}
    2423   runid=${primaries[$s+$s+1]}
     24   # work around for 100 missing old files to avoid warnings every hour
    2525   currentnight=`date +%Y%m%d`
    26    checknight=`echo " $currentnight - $transferdelay " | bc -l`
    27    # work around for 100 missing old files to avoid warnings every hour
    2826   checknight2=`echo " $currentnight - 365 - $transferdelay " | bc -l`
    2927   
    … …  
    6260                            ;;
    6361                       *)   # print warning only for files which are older than $transferdelay days
    64                             if [ $night -lt $checknight ]
     62                            if [ $night -lt $checknight ] && [ $night -gt $checknight2 ]
    6563                            then
    6664                               printprocesslog "WARN "$drsfile" and "$failsdrsfile" missing."

  • trunk/DataCheck/Processing/FillAuxCamHum.sh

    r18321 r18708  
    3535   getdates 6
    3636fi
     37
    3738
    3839
    … …  
    9899      if ! [ -e $rawfile ]
    99100      then
    100          printprocesslog "ERROR: "$rawfile" not found."
     101         if [ $runnumber -lt $checknight ]
     102         then
     103            printprocesslog "WARN "$rawfile" not found."
     104         else
     105            printprocesslog "INFO "$rawfile" not found."
     106         fi
    101107         continue
    102108      fi
    … …  
    111117      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    112118      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    113       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     119      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    114120      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    115       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     121      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    116122      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    117123      then

  • trunk/DataCheck/Processing/FillAuxContTemp.sh

    r18321 r18708  
    3535   getdates 6
    3636fi
     37
     38
    3739
    3840printprocesslog "INFO processing the following night(s): "${dates[@]}
    … …  
    101103      if ! [ -e $rawfile ]
    102104      then
    103          printprocesslog "ERROR: "$rawfile" not found."
     105         if [ $runnumber -lt $checknight ]
     106         then
     107            printprocesslog "WARN "$rawfile" not found."
     108         else
     109            printprocesslog "INFO "$rawfile" not found."
     110         fi
    104111         continue
    105112      fi
    … …  
    107114      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    108115      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    109       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     116      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    110117      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    111       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     118      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    112119      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    113120      then

  • trunk/DataCheck/Processing/FillAuxCtrDev.sh

    r18321 r18708  
    3535   getdates 6
    3636fi
     37
     38
    3739
    3840printprocesslog "INFO processing the following night(s): "${dates[@]}
    … …  
    100102      if ! [ -e $rawfile ]
    101103      then
    102          printprocesslog "ERROR: "$rawfile" not found."
     104         if [ $runnumber -lt $checknight ]
     105         then
     106            printprocesslog "WARN "$rawfile" not found."
     107         else
     108            printprocesslog "INFO "$rawfile" not found."
     109         fi
    103110         continue
    104111      fi
    … …  
    106113      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    107114      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    108       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     115      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    109116      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    110       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     117      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    111118      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    112119      then

  • trunk/DataCheck/Processing/FillAuxCurrents.sh

    r18321 r18708  
    3535   getdates 6
    3636fi
     37
     38
    3739
    3840printprocesslog "INFO processing the following night(s): "${dates[@]}
    … …  
    153155      if ! [ -e $rawfile ]
    154156      then
    155          printprocesslog "ERROR: "$rawfile" not found."
     157         if [ $runnumber -lt $checknight ]
     158         then
     159            printprocesslog "WARN "$rawfile" not found."
     160         else
     161            printprocesslog "INFO "$rawfile" not found."
     162         fi
    156163         continue
    157164      fi
    … …  
    159166      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    160167      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    161       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     168      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    162169      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    163       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     170      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    164171      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    165172      then
    166          printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
     173         printprocesslog "WARN "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
    167174         continue
    168175      fi

  • trunk/DataCheck/Processing/FillAuxData.sh

    r18698 r18708  
    3636   getdates 3 7 19
    3737fi
     38
    3839
    3940
    … …  
    181182      if ! [ -e $rawfile ]
    182183      then
    183          printprocesslog "ERROR: "$rawfile" not found."
     184         if [ $night -lt $checknight ]
     185         then
     186            printprocesslog "WARN "$rawfile" not found."
     187         else
     188            printprocesslog "INFO "$rawfile" not found."
     189         fi
    184190         continue
    185191      fi

  • trunk/DataCheck/Processing/FillAuxTemp.sh

    r18321 r18708  
    3636fi
    3737
     38
     39
    3840printprocesslog "INFO processing the following night(s): "${dates[@]}
    3941echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
    … …  
    109111      if ! [ -e $rawfile ]
    110112      then
    111          printprocesslog "ERROR: "$rawfile" not found."
     113         if [ $runnumber -lt $checknight ]
     114         then
     115            printprocesslog "WARN "$rawfile" not found."
     116         else
     117            printprocesslog "INFO "$rawfile" not found."
     118         fi
    112119         continue
    113120      fi
    … …  
    122129      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    123130      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    124       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     131      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    125132      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    126       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     133      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    127134      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    128135      then

  • trunk/DataCheck/Processing/FillAuxThresholds.sh

    r18321 r18708  
    3737fi
    3838
     39
     40
    3941printprocesslog "INFO processing the following night(s): "${dates[@]}
    4042echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
    … …  
    110112      if ! [ -e $rawfile ]
    111113      then
    112          printprocesslog "ERROR: "$rawfile" not found."
     114         if [ $runnumber -lt $checknight ]
     115         then
     116            printprocesslog "WARN "$rawfile" not found."
     117         else
     118            printprocesslog "INFO "$rawfile" not found."
     119         fi
    113120         continue
    114121      fi
    … …  
    116123      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    117124      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    118       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     125      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    119126      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    120       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     127      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    121128      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    122129      then

  • trunk/DataCheck/Processing/FillDrsTemp.sh

    r18321 r18708  
    3535   getdates 6
    3636fi
     37
    3738
    3839
    … …  
    99100      if ! [ -e $rawfile ]
    100101      then
    101          printprocesslog "ERROR: "$rawfile" not found."
     102         if [ $runnumber -lt $checknight ]
     103         then
     104            printprocesslog "WARN "$rawfile" not found. "$night" "$checknight
     105         else
     106            printprocesslog "INFO "$rawfile" not found."
     107         fi
    102108         continue
    103109      fi
    … …  
    112118      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    113119      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    114       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     120      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    115121      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    116       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     122      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    117123      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    118124      then

  • trunk/DataCheck/Processing/FillEffectiveOn.sh

    r18321 r18708  
    3737
    3838
     39
    3940printprocesslog "INFO processing the following night(s): "${dates[@]}
    4041echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
    … …  
    9596      if ! [ -e $rawfile ]
    9697      then
    97          printprocesslog "ERROR: "$rawfile" not found."
     98         if [ $runnumber -lt $checknight ]
     99         then
     100            printprocesslog "WARN "$rawfile" not found."
     101         else
     102            printprocesslog "INFO "$rawfile" not found."
     103         fi
    98104         continue
    99105      fi
    … …  
    101107      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    102108      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    103       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     109      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    104110      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    105       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     111      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    106112      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    107113      then

  • trunk/DataCheck/Setup/setup.fact.isdc

    r18683 r18708  
    107107# further wishlist: adapt ratio automatically
    108108
     109# setup for warning/errors
     110#   print warnings/errors (eg for missing files) only after a delay of few days to account for transfer
     111transferdelay=3 #days
     112checknight=`date +%Y%m%d  --date="-${transferdelay}day"`
     113
    109114# setup transfer
    110115numrsyncwuelimit=3

  • trunk/DataCheck/Setup/setup.fact.lp.data

    r18694 r18708  
    9595resulttable2="AnalysisResultsNightLP"
    9696firstnight=20121213
     97
     98# setup for warning/errors
     99#   print warnings/errors (eg for missing files) only after a delay of few days to account for transfer
     100transferdelay=3 #days
     101checknight=`date +%Y%m%d  --date="-${transferdelay}day"`
     102
Note: See TracChangeset for help on using the changeset viewer.