Changeset 14155 for trunk/DataCheck/Processing

Timestamp:

06/11/12 18:28:21 (13 years ago)

Author:

Daniela Dorner

Message:

bugfix for drsstep; better handling of corrupted files; included new version of checksums

File:

• trunk/DataCheck/Processing/CheckRawData.sh (modified) (9 diffs)

trunk/DataCheck/Processing/CheckRawData.sh

@@ -6 +6 @@
 #   this causes a delay until files are in the database
 #   because they have to be rsynced and zipped first (see RsyncRawData.sh, ZipRawData.sh)
-
-# missing
-# entry in status table
+# 
+# starting from 2012/06/05 the keywords CHECKSUM and DATASUM are 
+#   available in the fits header of the rawfile
+#   starting from this time the md5sums are not calculated anymore
+#
+
+# todo
+# tempfile for fitsdump -h output
 
 # options: 
 skipmd5sum="no" # fill md5 sums in any case
 skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
-#skipmd5sum="yes" # do not fill md5 sums in any case
+skipmd5sum="yes" # do not fill md5 sums in any case #new default since 2012/06/05 when the checksum is available in heaser
 
 doupdate="yes" # update all entries
@@ -107 +112 @@
    for file in $fitsgzfiles
    do
+      filecorrupt="no"
       printprocesslog "INFO checking file "$file
-      #echo "INFO checking file "$file
+      #echo "INFO checking file "$file" at "`date`
 
       # raw and original file
+      # file: /loc_data/zipraw
+      # rawfile: /loc_data/raw
+      # origfile: /daq/raw
       rawfile=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
       origfile=`echo $rawfile | sed -e 's/loc_data/daq/'`
@@ -185 +194 @@
       then
          printprocesslog "WARN "$rawfile" has "$numfitserrors" fitserror(s). "
+         fitsdumperrors=`$factpath/fitsdump -h -t Events $file 2>&1 | grep corrupted`
+         if [ "$fitsdumperrors" != "" ]
+         then
+            filecorrupt="yes"
+         fi
          #numfitserrors=1
       fi
@@ -190 +204 @@
       numdrsfiles=
       step=
-      if [ $numfitserrors -eq 0 ]
+      #if [ $numfitserrors -eq 0 ]
+      if [ "$filecorrupt" == "no" ]
       then 
          # get run and file number from file
@@ -207 +222 @@
          fi
       
+         # get checksums from header
+         checksum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
+         if [ "$checksum" == "" ]
+         then 
+            printprocesslog "WARN checksum for file "$file" is empty."
+         fi
+         datasum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
+         if [ "$datasum" == "" ]
+         then 
+            printprocesslog "WARN datasum for file "$file" is empty."
+         fi
+         
          # check if this run has drs file
          #   in case file is available, get STEP from header
@@ -216 +243 @@
          if [ "$drscalib" == "T" ]
          then 
-            step=`$factpath/fitsdump -h -t Events $drsfile  2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
+            step=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep DRSSTEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
+            stepfromdrs=`$factpath/fitsdump -h -t Events $drsfile  2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
+            if [ "$stepfromdrs" != "$step" ]
+            then
+               printprocesslog "ERROR for file "$file" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree."
+               if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ]
+               then 
+                  step=$stepfromdrsfile
+                  printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$file"."
+               fi
+            fi
             if ! [ $numdrsfiles -eq 1 ]
             then 
                printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
+            fi
+            if [ "$step" = "" ]
+            then 
+               printprocesslog "ERROR file "$file" has drsfiles ("$numdrsfiles"), but step ("$step") is empty."
             fi
          else
@@ -362 +403 @@
       fi
       query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
-      if [ $numfitserrors -eq 0 ]
+      #if [ $numfitserrors -eq 0 ]
+      if [ "$filecorrupt" == "no" ]
       then 
          query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'"
@@ -410 +452 @@
          fi
       fi
+      if [ "$checksum" != "" ]
+      then
+         query=$query", fCheckSum='"$checksum"'"
+      fi
+      if [ "$datasum" != "" ]
+      then
+         query=$query", fDataSum='"$datasum"'"
+      fi
       if [ "$md5sum" != "" ]
       then
@@ -433 +483 @@
       query=$query" "$querystop
       # send query to DB
+      #echo $query
       sendquery >/dev/null
    done