Ignore:
Timestamp:
06/11/12 18:28:21 (12 years ago)
Author:
Daniela Dorner
Message:
bugfix for drsstep; better handling of corrupted files; included new version of checksums
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Processing/CheckRawData.sh

    r13601 r14155  
    66#   this causes a delay until files are in the database
    77#   because they have to be rsynced and zipped first (see RsyncRawData.sh, ZipRawData.sh)
    8 
    9 # missing
    10 # entry in status table
     8#
     9# starting from 2012/06/05 the keywords CHECKSUM and DATASUM are
     10#   available in the fits header of the rawfile
     11#   starting from this time the md5sums are not calculated anymore
     12#
     13
     14# todo
     15# tempfile for fitsdump -h output
    1116
    1217# options:
    1318skipmd5sum="no" # fill md5 sums in any case
    1419skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
    15 #skipmd5sum="yes" # do not fill md5 sums in any case
     20skipmd5sum="yes" # do not fill md5 sums in any case #new default since 2012/06/05 when the checksum is available in heaser
    1621
    1722doupdate="yes" # update all entries
    … …  
    107112   for file in $fitsgzfiles
    108113   do
     114      filecorrupt="no"
    109115      printprocesslog "INFO checking file "$file
    110       #echo "INFO checking file "$file
     116      #echo "INFO checking file "$file" at "`date`
    111117
    112118      # raw and original file
     119      # file: /loc_data/zipraw
     120      # rawfile: /loc_data/raw
     121      # origfile: /daq/raw
    113122      rawfile=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
    114123      origfile=`echo $rawfile | sed -e 's/loc_data/daq/'`
    … …  
    185194      then
    186195         printprocesslog "WARN "$rawfile" has "$numfitserrors" fitserror(s). "
     196         fitsdumperrors=`$factpath/fitsdump -h -t Events $file 2>&1 | grep corrupted`
     197         if [ "$fitsdumperrors" != "" ]
     198         then
     199            filecorrupt="yes"
     200         fi
    187201         #numfitserrors=1
    188202      fi
    … …  
    190204      numdrsfiles=
    191205      step=
    192       if [ $numfitserrors -eq 0 ]
     206      #if [ $numfitserrors -eq 0 ]
     207      if [ "$filecorrupt" == "no" ]
    193208      then
    194209         # get run and file number from file
    … …  
    207222         fi
    208223     
     224         # get checksums from header
     225         checksum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
     226         if [ "$checksum" == "" ]
     227         then
     228            printprocesslog "WARN checksum for file "$file" is empty."
     229         fi
     230         datasum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
     231         if [ "$datasum" == "" ]
     232         then
     233            printprocesslog "WARN datasum for file "$file" is empty."
     234         fi
     235         
    209236         # check if this run has drs file
    210237         #   in case file is available, get STEP from header
    … …  
    216243         if [ "$drscalib" == "T" ]
    217244         then
    218             step=`$factpath/fitsdump -h -t Events $drsfile  2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
     245            step=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DRSSTEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
     246            stepfromdrs=`$factpath/fitsdump -h -t Events $drsfile  2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
     247            if [ "$stepfromdrs" != "$step" ]
     248            then
     249               printprocesslog "ERROR for file "$file" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree."
     250               if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ]
     251               then
     252                  step=$stepfromdrsfile
     253                  printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$file"."
     254               fi
     255            fi
    219256            if ! [ $numdrsfiles -eq 1 ]
    220257            then
    221258               printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
     259            fi
     260            if [ "$step" = "" ]
     261            then
     262               printprocesslog "ERROR file "$file" has drsfiles ("$numdrsfiles"), but step ("$step") is empty."
    222263            fi
    223264         else
    … …  
    362403      fi
    363404      query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
    364       if [ $numfitserrors -eq 0 ]
     405      #if [ $numfitserrors -eq 0 ]
     406      if [ "$filecorrupt" == "no" ]
    365407      then
    366408         query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'"
    … …  
    410452         fi
    411453      fi
     454      if [ "$checksum" != "" ]
     455      then
     456         query=$query", fCheckSum='"$checksum"'"
     457      fi
     458      if [ "$datasum" != "" ]
     459      then
     460         query=$query", fDataSum='"$datasum"'"
     461      fi
    412462      if [ "$md5sum" != "" ]
    413463      then
    … …  
    433483      query=$query" "$querystop
    434484      # send query to DB
     485      #echo $query
    435486      sendquery >/dev/null
    436487   done
Note: See TracChangeset for help on using the changeset viewer.