Changeset 18083


Ignore:
Timestamp:
01/04/15 05:41:02 (10 years ago)
Author:
Daniela Dorner
Message:
updated to let the scripts run on newdaq
Location:
trunk/DataCheck/Processing
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Processing/CheckRawData.sh

    r17907 r18083  
    11#!/bin/bash
    22
    3 # this script has been written to run on La Palma on the machine data
    4 #   i.e. paths are only working on this machine
    5 # the script starts from the zipped files
    6 #   this causes a delay until files are in the database
    7 #   because they have to be rsynced and zipped first (see RsyncRawData.sh, ZipRawData.sh)
    8 #
    9 # starting from 2012/06/05 the keywords CHECKSUM and DATASUM are
    10 #   available in the fits header of the rawfile
    11 #   starting from this time the md5sums are not calculated anymore (not true, only from 2012/12/17)
    12 #
     3# completely new version of the script
     4# running on newdaq to fill the information as fast as possible
     5# ftools is not used in this script anymore
    136
    147# todo
    158# tempfile for fitsdump -h output
    16 
    17 ## options:
    18 #skipmd5sum="no" # fill md5 sums in any case
    19 #skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
    20 #skipmd5sum="yes" # do not fill md5 sums in any case #new default since 2012/06/05 when the checksum is available in heaser
    219
    2210# option whether to fill all row or only those where information is missing
    … …  
    2816fi
    2917
    30 
    31 root=/opt/root_svn/bin/thisroot.sh
    32 
    3318source `dirname $0`/../Sourcefile.sh
    3419printprocesslog "INFO starting $0 with options doupdate="$doupdate #" and skipmd5sum="$skipmd5sum
    35 
    36 # setup to use ftools
    37 source $HEADAS/headas-init.sh
    3820
    3921# check if software is available
    … …  
    4527
    4628# check if paths are available
    47 if ! ls /daq/raw >/dev/null 2>&1
     29if ! ls $rawdata >/dev/null 2>&1
    4830then
    49    printprocesslog "ERROR /daq/raw is not available."
    50    finish
    51 fi
    52 if ! ls /newdaq/raw >/dev/null 2>&1
    53 then
    54    printprocesslog "ERROR /newdaq/raw is not available."
    55    finish
    56 fi
    57 if ! ls /loc_data/zipraw >/dev/null 2>&1
    58 then
    59    printprocesslog "ERROR /loc_data/zipraw is not available."
     31   printprocesslog "ERROR "$rawdata" is not available."
    6032   finish
    6133fi
    … …  
    8153for date in ${dates[@]}
    8254do
    83    date2=`echo $date | sed -e 's/\///g'`
    84    night=
    85    query="SELECT fNight FROM AuxFilesAvailISDCStatus WHERE fNight="$date2
    86    night=`sendquery`
    87    if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    88    then
    89       printprocesslog "INFO insert "$date2" to AuxFilesAvailISDCStatus"
    90       query="INSERT AuxFilesAvailISDCStatus SET fNight="$date2", fPriority="$date2
    91       sendquery >/dev/null 2>&1
    92    fi
    93    night=
    94    query="SELECT fNight FROM DriveFileAvailISDCStatus WHERE fNight="$date2
    95    night=`sendquery`
    96    if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    97    then
    98       printprocesslog "INFO insert "$date2" to DriveFileAvailISDCStatus"
    99       query="INSERT DriveFileAvailISDCStatus SET fNight="$date2", fPriority="$date2
    100       sendquery >/dev/null 2>&1
    101    fi
    102    night=
    103    query="SELECT fNight FROM RatesFileAvailISDCStatus WHERE fNight="$date2
    104    night=`sendquery`
    105    if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    106    then
    107       printprocesslog "INFO insert "$date2" to RatesFileAvailISDCStatus"
    108       query="INSERT RatesFileAvailISDCStatus SET fNight="$date2", fPriority="$date2
    109       sendquery >/dev/null 2>&1
    110    fi
    111    night=
    112    query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$date2
    113    night=`sendquery`
    114    if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    115    then
    116       printprocesslog "INFO insert "$date2" to AuxDataInsertStatus"
    117       query="INSERT AuxDataInsertStatus SET fNight="$date2", fPriority="$date2
    118       sendquery >/dev/null 2>&1
    119    fi
    120    ziprawdir=/loc_data/zipraw/$date
    121    # check if data are available from that night
    122    if ! [ -d $ziprawdir ]
    123    then
    124       printprocesslog "INFO "$ziprawdir" does not exist."
    125       continue
    126    else
    127       printprocesslog "INFO processing "$ziprawdir"..."
    128    fi
    129 
    130    # find all fits.gz files starting with the oldest file
    131    printprocesslog "INFO finding files to be checked in $ziprawdir..."
    132    fitsgzfiles=`find $ziprawdir -type f -name '*.fits.gz'| sort `
    133 
    13455   # get runnumber from date
    13556   runnumber=`echo $date | sed -e 's/\///g'`
    13657   
     58   night=
     59   query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$runnumber
     60   night=`sendquery`
     61   if [ "$night" == "" ] && ls $rawdata/$date >/dev/null 2>&1
     62   then
     63      printprocesslog "INFO insert "$runnumber" to AuxDataInsertStatus"
     64      query="INSERT AuxDataInsertStatus SET fNight="$runnumber", fPriority="$runnumber
     65      sendquery >/dev/null 2>&1
     66   fi
     67   rawdir=$rawdata/$date
     68   # check if data are available from that night
     69   if ! [ -d $rawdir ]
     70   then
     71      printprocesslog "INFO "$rawdir" does not exist."
     72      continue
     73   else
     74      printprocesslog "INFO processing "$rawdir"..."
     75   fi
     76
     77   # find all fits.gz files starting with the oldest file
     78   printprocesslog "INFO finding files to be checked in $rawdir..."
     79   #fitsfiles=`ls $rawdir/*.fits 2>/dev/null | sort `
     80   fitsfiles=`find $rawdir -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort `
     81
    13782   # loop to check files
    138    for file in $fitsgzfiles
     83   for file in $fitsfiles
    13984   do
    140       filecorrupt="no"
    14185      printprocesslog "INFO checking file "$file
    142       #echo "INFO checking file "$file" at "`date`
    143 
    144       # raw and original file
    145       # file: /loc_data/zipraw
    146       # rawfile: /daq/raw
    147       # origfile: /newdaq/raw
    148       rawfile=`echo $file | sed -e 's/loc_data/daq/' -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
    149       rawfile2=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
    150       origfile=`echo $rawfile | sed -e 's/daq/newdaq/'`
     86      #echo "INFO checking file "$file
     87      filenotclosed="no"
    15188
    15289      # check if it is drs file
    … …  
    15895      fi
    15996
    160 # not needed anymore as files are only rsynced from newdaq to daq when they are closed
    161 #      # check if file is already finished
    162 #      # original file on daq (if data was taken on daq
    163 #      if [ -e $origfile ]
    164 #      then
    165 #         # check if raw file was changed in the last 30 minutes
    166 #         isnew=`find $origfile -cmin -30`
    167 #         if [ "$isnew" != "" ]
    168 #         then
    169 #            printprocesslog "WARN "$origfile" is not older than 30 min. -> continue"
    170 #            continue
    171 #         fi
    172 #
    173 #         # get time of last modification as seconds since Epoch for both files
    174 #         timeorig=`stat -c %Y $origfile`
    175 #         timecopy=`stat -c %Y $rawfile`
    176 #         # compare times
    177 #         if ! [ $timeorig -eq $timecopy ]
    178 #         then
    179 #            # if times are not the same, the file is still open => no check
    180 #            printprocesslog "INFO file "$rawfile" not yet closed. -> continue"
    181 #            continue
    182 #         fi
    183 #      else
    184 #         # if the origfile doesn't exist, the data was probably written not on daq but on data
    185 #         printprocesslog "INFO file "$rawfile" was probably taken on data and not daq."
    186 #      fi
    187      
    18897      # get run and file number form filename
    189       runnumbererror="no"
    190       numbererror="no"
    19198      numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
    19299      runnumberfromname=`echo $numberfromname | cut -d_ -f1`
    … …  
    194101      if [ "$runnumber" != "$runnumberfromname" ]
    195102      then
    196          runnumbererror="yes"
    197103         printprocesslog "ERROR for file "$file": runnumber from date ("$runnumber") and filename ("$runnumberfromname") don't agree."
    198104      fi
    … …  
    214120     
    215121      runtype=
    216       # check if fits file is corrupted
    217       numfitserrors=0
    218       #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
    219       #numfitserrors=`fverify $rawfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    220 # ftools missing on daq
    221 #      numfitserrors=`ssh daq "export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/ ; source $HEADAS/headas-init.sh ; fverify $rawfile2 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'"`
    222 #      echo $rawfile2" ------ "$numfitserrors
    223       #if [ "$checkfitsfile" == "" ]
    224       if [ $numfitserrors -gt 0 ]
    225       then
    226          printprocesslog "WARN "$rawfile2" has "$numfitserrors" fitserror(s). "
    227          #fitsdumperrors=`$factpath/fitsdump -h -t Events $file 2>&1 | grep corrupted`
    228          fitsdumperrors=`$factpath/fitsdump -h $file 2>&1 | grep corrupted`
    229          if [ "$fitsdumperrors" != "" ]
    230          then
    231             filecorrupt="yes"
    232          fi
    233          #numfitserrors=1
    234       fi
    235 
    236122      # check if file was closed properly
    237123      if [ "`echo $file | grep -o drs`" == "drs" ]
    238124      then
    239          nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
    240          tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
     125         nondrsfile=`echo $file | sed -e 's/[.]drs//g'`".fz" # fixme: maybe more flexible
     126         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
     127         checkstatus=$?
    241128      else
    242          tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
    243       fi
    244       if [ "$tstop" == "0" ]
    245       then
    246          echo $file" not yet closed."
    247          echo "WARN: $file has empty TSTOP"
    248          filecorrupt="yes"
     129         tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
     130         checkstatus=$?
     131      fi
     132      #if [ $checkstatus -gt 0 ]
     133      #then
     134      #   echo "WARN: getting tstop for file "$file" failed: tstop: -"$tstop"-"`date`
     135      #fi
     136      if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
     137      then
     138         fileaccessed=`find $file -amin -30`
     139         if ! [ "$fileaccessed" == "" ]
     140         then
     141            printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
     142            continue
     143         else
     144            printprocesslog "WARN "$file" has empty TSTOP but was not touched for 30 minutes"
     145            filenotclosed="yes"
     146         fi
    249147      fi
    250148
    251149      numdrsfiles=
    252150      step=
    253       #if [ $numfitserrors -eq 0 ]
    254       if [ "$filecorrupt" == "no" ]
     151      if [ "$filenotclosed" == "no" ]
    255152      then
    256153         # get run and file number from file
    257          #runnumberfromfile=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
    258154         runnumberfromfile=`$factpath/fitsdump -h $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
    259          #filenumberfromfileorig=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
    260155         filenumberfromfileorig=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
    261156         if [ "$runnumberfromfile" = "" ] || [ "$filenumberfromfileorig" = "" ]
    … …  
    267162         if [ "$numberfromfile" != "$numberfromname" ]
    268163         then
    269             numbererror="yes"
    270164            printprocesslog "ERROR for file "$file": number from filename ("$numberfromname") and file ("$numberfromfile") don't agree."
    271165         fi
    272166     
    273167         # get checksums from header
    274          #checksum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
    275168         checksum=`$factpath/fitsdump -h $file  2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
    276169         if [ "$checksum" == "" ]
    … …  
    278171            printprocesslog "WARN checksum for file "$file" is empty."
    279172         fi
    280          #datasum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
    281173         datasum=`$factpath/fitsdump -h $file  2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
    282174         if [ "$datasum" == "" ]
    … …  
    289181         # in the very beginning only drs-files were existing
    290182         # in the beginning the keywords DRSCALIB and STEP were not existing
    291          drsfile=`echo $file | sed -e 's/fits/drs.fits/'`
     183         drsfile=`echo $file | sed -e 's/fits.fz/drs.fits/'`
    292184         numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
    293          #drscalib=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DRSCALIB | grep -E -o "['][TF][']" | sed -e "s/'//g"`
    294185         drscalib=`$factpath/fitsdump -h $file  2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"`
    295186         if [ "$drscalib" == "T" ]
    296187         then
    297             #step=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DRSSTEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
    298188            step=`$factpath/fitsdump -h $file  2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"`
    299             #stepfromdrs=`$factpath/fitsdump -h -t Events $drsfile  2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
    300189            stepfromdrs=`$factpath/fitsdump -h $drsfile  2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"`
    301190            if [ "$stepfromdrs" != "$step" ]
    … …  
    322211            fi
    323212         fi
    324          #echo $numdrsfiles" "$stepfromdrs" "$step" "$drscalib" ("$file")"
    325213         
    326214         # get other variables from header
    327          #runtype=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
    328215         runtype=`$factpath/fitsdump -h $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
    329          #echo "runtype for file "$file": "$runtype
    330          #roi=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
    331216         roi=`$factpath/fitsdump -h $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
    332          #roitm=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
    333217         roitm=`$factpath/fitsdump -h $file  2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
    334          #numevents=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
    335218         numevents=`$factpath/fitsdump -h $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
    336          #numphys=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
    337219         numphys=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
    338          #numext1=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
    339220         numext1=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
    340          #numext2=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
    341221         numext2=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
    342          #numelp=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
    343222         numelp=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
    344          #numilp=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
    345223         numilp=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
    346          #numoth=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
    347224         numoth=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
    348          #numped=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
    349225         numped=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
    350          #numtime=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
    351226         numtime=`$factpath/fitsdump -h $file  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
    352          #compiled=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
    353227         compiled=`$factpath/fitsdump -h $file  2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
    354228         if ! [ "$compiled" == "" ]
    … …  
    358232            compiletime=
    359233         fi
    360          #revnum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
    361234         revnum=`$factpath/fitsdump -h $file  2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
    362235         # in newest data start time is in DATE-OBS
    363236         # in older data start time is in TSTART
    364237         # in the beginning TSTART was empty
    365          #runstart=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    366238         runstart=`$factpath/fitsdump -h $file  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    367          #runstart2=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    368239         runstart2=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    369240         if [ "$runstart" == ""  ]
    … …  
    371242            if [ "$runstart2" == ""  ]
    372243            then
    373                #runstart=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    374                runstart=`$factpath/fitsdump -h $file  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     244               # for very old data, but gives problems for newer data
     245               #runstart=`$factpath/fitsdump -h $file  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     246               runstart="0000-00-00 00:00:00"
    375247            else
    376248               runstart=$runstart2
    … …  
    380252         # in older data start time is in TSTOP
    381253         # in the beginning TSTOP was empty
    382          #runstop=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    383254         runstop=`$factpath/fitsdump -h $file  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    384          #runstop2=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    385255         runstop2=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    386256         if [ "$runstop" == ""  ]
    … …  
    388258            if [ "$runstop2" == ""  ]
    389259            then
    390                runstop=`stat $file  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
     260               # for very old data, but gives problems for newer data
     261               #runstop=`stat $file  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
     262               runstop="0000-00-00 00:00:00"
    391263            else
    392264               runstop=$runstop2
    … …  
    427299      query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
    428300      printprocesslog "DEBUG get run type from DB. QUERY:"$query
    429       #result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` )
    430301      result2=( `sendquery` )
    431302      if [ ${#result2} -eq 0 ]
    … …  
    434305         continue
    435306      fi
    436 
    437 #      # check if entry has already checksums
    438 #      query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
    439 #      query=$query" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
    440 #      printprocesslog "DEBUG check if md5sums are alreay in DB. QUERY:"$query
    441 #      #result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"`
    442 #      result5=`sendquery`
    443 #
    444 #      # get md5sums of raw and zip file
    445 #      #   to safe time for tests and update this can be skipped ($skipmd5sum)
    446 #      md5sum=
    447 #      md5sumzip=
    448 #      if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ]
    449 #      then
    450 #         #echo "skip: "$skipmd5sum >> $logfile 2>&1
    451 #         #echo "res5: -"$result5"-" >> $logfile 2>&1
    452 #         #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ]
    453 #         if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ]
    454 #         then
    455 #            #zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
    456 #            # only do the md5sum if the zipfile is already available
    457 #            if ls $file >/dev/null 2>&1
    458 #            then
    459 #               printprocesslog "INFO calculation md5sum for rawfile "$rawfile
    460 #               md5sum=`md5sum $rawfile | cut -d' ' -f1`
    461 #               printprocesslog "INFO calculation md5sum for zipfile "$file
    462 #               md5sumzip=`md5sum $file | cut -d' ' -f1`
    463 #            fi
    464 #         fi
    465 #      fi
    466307
    467308      # insert or update depending on whether run exists
    … …  
    477318      fi
    478319      query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
    479       #if [ $numfitserrors -eq 0 ]
    480       if [ "$filecorrupt" == "no" ]
     320      if [ "$filenotclosed" == "no" ]
    481321      then
    482322         query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'"
    … …  
    534374         query=$query", fDataSum='"$datasum"'"
    535375      fi
    536 #      if [ "$md5sum" != "" ]
    537 #      then
    538 #         query=$query", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
    539 #      fi
    540376      if [ "$numdrsfiles" != "" ]
    541377      then
    … …  
    554390         query=$query", fRevisionNumber='"$revnum"'"
    555391      fi
    556       query=$query", fFitsFileErrors="$numfitserrors
     392      #query=$query", fFitsFileErrors="$numfitserrors
    557393      query=$query" "$querystop
    558394      # send query to DB

  • trunk/DataCheck/Processing/FillAuxData.sh

    r17907 r18083  
    11#!/bin/bash
    22
    3 # missing:
    4 #   vferify of files
    5 #   only treat file if it is there
     3# new version of the script to run on newdaq
     4
     5# option whether to fill all row or only those where information is missing
     6# $doupdate might be given as environment variable
     7if [ "$doupdate" = "" ]
     8then
     9   doupdate="yes" # update all entries (needed when new fields have been added)
     10   doupdate="no" # fill only entries which are not yet existing (default)
     11fi
     12
     13source `dirname $0`/../Sourcefile.sh
     14printprocesslog "INFO starting $0 with option doupdate="$doupdate
     15
     16logfile=$runlogpath"/FillAuxData-"$datetime".log"
     17date >> $logfile
     18
     19# check if software is available
     20if ! ls $factpath/fitsdump >/dev/null 2>&1
     21then
     22   printprocesslog "ERROR "$factpath"/fitsdump is not available."
     23   finish
     24fi
     25
     26# get dates
     27if [ "$certaindate" != "" ]
     28then
     29   checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
     30   if [ "$checkstring" = "" ]
     31   then
     32      echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
     33      finish
     34   fi
     35   getdates $certaindate
     36else
     37   # get all night
     38   #getdates "all"
     39   # get last 3 nights if hour between 7 and 19h, else only current night
     40   getdates 3 7 19
     41fi
     42
     43
     44printprocesslog "INFO processing the following night(s): "${dates[@]}
     45echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
    646
    747# mjd of 1970-01-01
    … …  
    2060# trigger rate has as first value -1, but with using the median it should be fine
    2161
    22 # option whether to fill all row or only those where information is missing
    23 # $doupdate might be given as environment variable
    24 if [ "$doupdate" = "" ]
    25 then
    26    doupdate="yes" # update all entries (needed when new fields have been added)
    27    doupdate="no" # fill only entries which are not yet existing (default)
    28 fi
    29 
    30 source `dirname $0`/../Sourcefile.sh
    31 printprocesslog "INFO starting $0 with option doupdate="$doupdate
    32 
    33 logfile=$runlogpath"/FillAuxData-"$datetime".log"
    34 date >> $logfile
    35 
    36 function getfitsstatistics()
     62
     63function evaluatestatistics()
    3764{
    38    # $1 filename
    39    # $2 colname
    40    # $3 tstart
    41    # $4 tstop
    42    good=
     65   # $1 variable name
     66   # $@ statistics
     67   if [ "$2" = "" ]
     68   then
     69      printprocesslog "WARN couldn't get statistics from file $1 for run "$date" "$file
     70      continue
     71   fi
    4372   min=
    4473   mean=
    45    median=
     74   med=
    4675   max=
    47    tmpfile=`dirname $0`/`basename $1`.tmp
    48    echo "ftcopy $1'[col Time]' - | ftstat - | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'" >> $logfile
    49    timefromfile=`ftcopy $1'[col Time]' - 2>>$logfile | ftstat - 2>>$logfile | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'`
    50    echo "here-"$timefromfile"-" >> $logfile
    51    if [ "$timefromfile" == "" ]
    52    then
    53       printprocesslog "WARN couldn't get time from file "$1
    54       echo "WARN couldn't get time from file "$1
    55       continue
    56    fi
    57    if [ $timefromfile -gt 30000 ]
    58    then
    59       echo "ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile
    60       #ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -
    61       ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile
    62    else
    63       echo "ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile
    64       #ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -
    65       ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile
    66    fi
    67    good=`cat $tmpfile | grep 'good' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
    68    min=`cat $tmpfile | grep 'min' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
    69    mean=`cat $tmpfile | grep 'mean' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
    70    median=`cat $tmpfile | grep 'median' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
    71    max=`cat $tmpfile | grep 'max' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
    72    sigma=`cat $tmpfile | grep 'sigma' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
    73    if [ "$good" = "" ]
    74    then
    75       printprocesslog "WARN couldn't get statistics from file $1 for run $date $file"
    76       rm $tmpfile
    77       continue
    78       #finish
    79    fi
    80    #echo "good: "$good
    81    #echo "min: "$min
    82    #echo "max: "$max
    83    #echo "mean: "$max
    84    #echo "median: "$max
    85    rm $tmpfile
     76   rms=
     77   evaluation=`echo $@ | grep -E -o '\['${1}':0[:]?[0-9]*\]\ Min:\ [-]?[0-9]+[.]?[0-9]*\ Max:\ [-]?[0-9]+[.]?[0-9]*\ Med:\ [-]?[0-9]+[.]?[0-9]*\ Avg:\ [-]?[0-9]+[.]?[0-9]*\ Rms:\ [-]?[0-9]+[.]?[0-9]*[e]?[-]?[0-9]*'`
     78   if [ "$evaluation" = "" ]
     79   then
     80      printprocesslog "WARN empty evaluation of statistic ("$@") for run "$date" "$file
     81      #echo "   ---> "$@
     82   fi
     83   #echo "ev: "$evaluation
     84   min=`echo $evaluation | grep -E -o 'Min:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Min:\ //'`
     85   max=`echo $evaluation | grep -E -o 'Max:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Max:\ //'`
     86   med=`echo $evaluation | grep -E -o 'Med:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Med:\ //'`
     87   mean=`echo $evaluation | grep -E -o 'Avg:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Avg:\ //'`
     88   rms=`echo $evaluation | grep -E -o 'Rms:\ [-]?[0-9]+[.]?[0-9]*[e]?[-]?[0-9]+' | sed -e 's/Rms:\ //'`
     89   #echo "eval: "$min" "$max" "$med" "$mean" "$rms
    8690}
    87 
    88 # setup to use ftools
    89 source $HEADAS/headas-init.sh
    90 
    91 # check if software is available
    92 if ! ls $factpath/fitsdump >/dev/null 2>&1
    93 then
    94    printprocesslog "ERROR "$factpath"/fitsdump is not available."
    95    finish
    96 fi
    97 
    98 # get dates
    99 if [ "$certaindate" != "" ]
    100 then
    101    checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
    102    if [ "$checkstring" = "" ]
    103    then
    104       echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
    105       finish
    106    fi
    107    getdates $certaindate
    108 else
    109    # get all night
    110    #getdates "all"
    111    # get last 3 nights if hour between 7 and 19h, else only current night
    112    getdates 3 7 19
    113 fi
    114 
    115 printprocesslog "INFO processing the following night(s): "${dates[@]}
    116 echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
    11791
    11892# do filling of aux data
    … …  
    12094do
    12195   auxdir=$auxdata/$date
     96   rawdir=$rawdata/$date
    12297   runnumber=`echo $date | sed -e 's/\///g'`
    123 
    124    # get file numbers from DB
    125    #   but only for not-corrupted files
    126    query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
    127    # only runs which are not yet filled
    128    if [ "$doupdate" = "no" ]
    129    then
    130       query=$query" AND ISNULL(fRightAscension) "
    131    fi
    132    printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
    133    filenumbers=( `sendquery $query` )
    134    # proceed only if there are files available
    135    if [ ${#filenumbers} -eq 0 ]
    136    then
    137       printprocesslog "INFO No files found in the DB for night "$date
    138       continue
    139    fi
    14098
    14199   # check if aux files are available from that night
    … …  
    146104   else
    147105      printprocesslog "INFO processing files in "$auxdir
     106      #echo "INFO processing files in "$auxdir
     107   fi
     108
     109   # check if raw files are available from that night
     110   #   only needed to get start/stop time
     111   #   might be removed once the start/stop time comes from the DB
     112   if ! [ -d $rawdir ]
     113   then
     114      printprocesslog "INFO no data available in "$rawdir" -> continue"
     115      continue
     116   fi
     117
     118   # get file numbers from DB
     119   #   but only for not-corrupted files
     120   # as aux files are written only once a minute, select only files which are older than 1.5 minutes
     121   query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 AND NOT ISNULL(fRunStop) AND fRunStop < SUBTIME(UTC_TIMESTAMP(), \"00:01:30\")"
     122   # only runs which are not yet filled
     123   if [ "$doupdate" = "no" ]
     124   then
     125      query=$query" AND ISNULL(fRightAscension) "
     126   fi
     127   printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
     128   filenumbers=( `sendquery $query` )
     129   if [ ${#filenumbers} -eq 0 ]
     130   then
     131      printprocesslog "INFO No files found in the DB for night "$date
     132      continue
    148133   fi
    149134
    … …  
    153138   then
    154139      printprocesslog "WARN "$trackingfile" not found."
    155    else
    156       tracknumerrors=`fverify $trackingfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    157       if [ $tracknumerrors -gt 0 ]
    158       then
    159          printprocesslog "WARN for $trackingfile fverify returned "$tracknumerrors" error(s)."
    160       fi
    161140   fi
    162141   
    … …  
    166145      printprocesslog "WARN "$sourceposfile" not found."
    167146   else
    168       sourceposnumerrors=`fverify $sourceposfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    169       if [ $sourceposnumerrors -gt 0 ]
    170       then
    171          printprocesslog "WARN for $sourceposfile fverify returned "$sourceposnumerrors" error(s)."
    172       fi
    173147      sourceposfiletstarti=`$factpath/fitsdump -h $sourceposfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    174148      sourceposfiletstartf=`$factpath/fitsdump -h $sourceposfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
    … …  
    179153         sourceposfiletstart=`echo " $sourceposfiletstarti + $sourceposfiletstartf " | bc -l`
    180154      fi
    181       #echo "tstart: "$sourceposfiletstart
    182155   fi
    183156   
    … …  
    186159   then
    187160      printprocesslog "WARN "$triggerratefile" not found."
    188    else
    189       trignumerrors=`fverify $triggerratefile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    190       if [ $trignumerrors -gt 0 ]
    191       then
    192          printprocesslog "WARN for $triggerratefile fverify returned "$trignumerrors" error(s)."
    193       fi
    194161   fi
    195162   
    … …  
    198165   then
    199166      printprocesslog "WARN "$thresholdfile" not found."
    200    else
    201       treshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    202       if [ $treshnumerrors -gt 0 ]
    203       then
    204          printprocesslog "WARN for $thresholdfile fverify returned "$treshnumerrors" error(s)."
    205       fi
    206167   fi
    207168   
    … …  
    210171   then
    211172      printprocesslog "WARN "$biasvoltagefile" not found."
    212    else
    213       biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    214       if [ $biasnumerrors -gt 0 ]
    215       then
    216          printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)."
    217       fi
    218173   fi
    219174
    … …  
    223178      printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
    224179      echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
     180      #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`
     181     
    225182      # get information from rawfile
    226       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
     183      rawfile=`ls $rawdir/$runnumber"_"\`printf %03d $filenum\`.fits*`
    227184      if ! [ -e $rawfile ]
    228185      then
    … …  
    230187         continue
    231188      fi
    232       #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
    233       #if [ "$checkfitsfile" == "" ]
    234       #then
    235       #   numfitserrors=1
    236       #   printprocesslog "WARN: "$rawfile" probably corrupted."
    237       #   continue
    238       #fi
    239       runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
    240       mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
     189
     190      #runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
     191      #mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    241192      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    242193      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
    … …  
    277228
    278229      # get information from source_pos file
    279       if [ -e $sourceposfile ] && [ $sourceposnumerrors -eq 0 ]
     230      if [ -e $sourceposfile ]
    280231      then
    281232         sourcename=`$factpath/fitsdump ${sourceposfile} -c Time -c Name --filter='[1]<'${tstop}' && [1]>'${sourceposfiletstart}  2>/dev/null | tail -1 2>&1 | grep -o -E "['][a-zA-Z0-9\ \.\+\-]+[']" | sed -e "s/'//g"`
    … …  
    295246      # build query to update runinfo in DB
    296247      query="UPDATE RunInfo SET "
     248     
    297249      # fill source key only if available
    298250      if ! [ "$sourcekey" = "" ]
    … …  
    304256     
    305257      # get information from tracking
    306       if [ -e $trackingfile ] && [ $tracknumerrors -eq 0 ]
    307       then
     258      if [ -e $trackingfile ]
     259      then
     260         # get statistics
     261         trackingstats=`$factpath/fitsdump $trackingfile -s -c Time -c Ra -c Dec -c Zd -c Az --filter='[1]<'${tstop}' && [1]>'${tstart}  2>/dev/null`
    308262         # RA
    309          getfitsstatistics $trackingfile "Ra" $tstart $tstop
    310          if [ "$min" == "$max" ] && [ $good -gt 0 ]
    311          then
    312             query=$query" fRightAscension="$mean
    313          else
    314             query=$query" fRightAscension=NULL"
    315             if [ $good -gt 0 ]
    316             then
     263         evaluatestatistics "Ra" $trackingstats
     264         #echo $min"_"$max"_"$
     265         if [ "$evaluation" != "" ]
     266         then
     267            if [ "$min" == "$max" ]
     268            then
     269               query=$query" fRightAscension="$mean
     270            else
     271               query=$query" fRightAscension=NULL"
    317272               printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")."
    318273            fi
    319          fi
    320          # Declination
    321          getfitsstatistics $trackingfile "Dec" $tstart $tstop
    322          if [ "$decmin" == "$decmax" ] && [ $good -gt 0 ]
    323          then
    324             query=$query", fDeclination="$mean
    325          else
    326             query=$query", fDeclination=NULL"
    327             if [ $good -gt 0 ]
    328             then
     274            # Declination
     275            evaluatestatistics "Dec" $trackingstats
     276            if [ "$decmin" == "$decmax" ]
     277            then
     278               query=$query", fDeclination="$mean
     279            else
     280               query=$query", fDeclination=NULL"
    329281               printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")."
    330282            fi
     283         else
     284            query=$query" fRightAscension=NULL"
     285            query=$query", fDeclination=NULL"
    331286         fi
    332287         # Zd
    333          getfitsstatistics $trackingfile "Zd" $tstart $tstop
    334          if [ $good -gt 0 ]
     288         evaluatestatistics "Zd" $trackingstats
     289         if [ "$evaluation" != "" ]
    335290         then
    336291            query=$query", fZenithDistanceMin="$min
    … …  
    343298         fi
    344299         # Az
    345          getfitsstatistics $trackingfile "Az" $tstart $tstop
    346          if [ $good -gt 0 ]
     300         evaluatestatistics "Az" $trackingstats
     301         if [ "$evaluation" != "" ]
    347302         then
    348303            query=$query", fAzimuthMin="$min
    … …  
    366321   
    367322      # get information from trigger
    368       if [ -e $triggerratefile ] && [ $trignumerrors -eq 0 ]
    369       then
    370          #echo " $triggerratefile TriggerRate $tstart $tstop"
    371          getfitsstatistics $triggerratefile "TriggerRate" $tstart $tstop
    372 #         if [ "$mjdreftrig" == "" ]
    373 #         then
    374 #            ratemin=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    375 #            ratemax=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    376 #            ratemean=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    377 #            ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    378 #         else
    379 #            ratemin=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    380 #            ratemax=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    381 #            ratemean=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    382 #            ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    383 #         fi
    384          if [ $good -gt 0 ]
    385          then
    386             query=$query", fTriggerRateMedian="$median
     323      if [ -e $triggerratefile ]
     324      then
     325         # get statistics
     326         triggerstats=`$factpath/fitsdump $triggerratefile -s -c Time -c TriggerRate --filter='[1]<'${tstop}' && [1]>'${tstart}  2>/dev/null`
     327         evaluatestatistics "TriggerRate" $triggerstats
     328         if [ "$evaluation" != "" ]
     329         then
     330            query=$query", fTriggerRateMedian="$med
    387331         else
    388332            query=$query", fTriggerRateMedian=NULL"
    … …  
    393337     
    394338      # get information from thresholds
    395       if [ -e $thresholdfile ] && [ $treshnumerrors -eq 0 ]
    396       then
    397          getfitsstatistics $thresholdfile "PatchThresh" $tstart $tstop
    398          if [ $good -eq 0 ]
    399          then
    400             getfitsstatistics $thresholdfile "PatchThresh" $tstart2 $tstop
    401          fi
    402          if [ $good -gt 0 ]
    403          then
    404             query=$query", fThresholdMedian="$median
     339      if [ -e $thresholdfile ]
     340      then
     341         # get statistics
     342         thresholdstats=`$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart}  2>/dev/null`
     343         evaluatestatistics "PatchThresh" $thresholdstats
     344         if [ "$evaluation" = "" ]
     345         then
     346            thresholdstats=`$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart2}  2>/dev/null`
     347            #echo "$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart2}  2>/dev/null"
     348            evaluatestatistics "PatchThresh" $thresholdstats
     349         fi
     350         if [ "$evaluation" != "" ]
     351         then
     352            query=$query", fThresholdMedian="$med
    405353         else
    406354            query=$query", fThresholdMedian=NULL"
    … …  
    411359
    412360      # get information from bias: U
    413       if [ -e $biasvoltagefile ] && [ $biasnumerrors -eq 0 ]
     361      if [ -e $biasvoltagefile ]
    414362      then
    415363         if [ $runnumber -gt 20120324 ]
    416364         then
    417              value="Uout"
    418          else
    419              value="U"
    420          fi
    421          getfitsstatistics $biasvoltagefile $value $tstart $tstop
    422          if [ $good -eq 0 ]
    423          then
    424             getfitsstatistics $biasvoltagefile $value $tstart2 $tstop
    425          fi
    426          if [ $good -gt 0 ]
    427          then
    428             query=$query", fBiasVoltageMedian="$median
     365            biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c Uout --filter='[1]<'${tstop}' && [1]>'${tstart}  2>/dev/null`
     366            evaluatestatistics "Uout" $biasstats
     367            if [ "$evaluation" = "" ]
     368            then
     369               biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c Uout --filter='[1]<'${tstop}' && [1]>'${tstart2}  2>/dev/null`
     370               evaluatestatistics "Uout" $biasstats
     371            fi
     372         else
     373            biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c U --filter='[1]<'${tstop}' && [1]>'${tstart}  2>/dev/null`
     374            evaluatestatistics "U" $biasstats
     375            if [ "$evaluation" = "" ]
     376            then
     377               biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c U --filter='[1]<'${tstop}' && [1]>'${tstart2}  2>/dev/null`
     378               evaluatestatistics "U" $biasstats
     379            fi
     380         fi
     381         if [ "$evaluation" != "" ]
     382         then
     383            query=$query", fBiasVoltageMedian="$med
    429384         else
    430385            query=$query", fBiasVoltageMedian=NULL"
Note: See TracChangeset for help on using the changeset viewer.