Changeset 14838 for trunk/DataCheck/Processing/CheckRawData.sh

Timestamp:

02/04/13 06:00:02 (12 years ago)

Author:

Daniela Dorner

Message:

adapted to new structure in LP (newdaw): removed md5sum, changed method for verfying that file is finished (check if file was closed via TSTOP), create directories only if data available

File:

• trunk/DataCheck/Processing/CheckRawData.sh (modified) (13 diffs)

trunk/DataCheck/Processing/CheckRawData.sh

@@ -9 +9 @@
 # starting from 2012/06/05 the keywords CHECKSUM and DATASUM are 
 #   available in the fits header of the rawfile
-#   starting from this time the md5sums are not calculated anymore
+#   starting from this time the md5sums are not calculated anymore (not true, only from 2012/12/17)
 #
 
@@ -15 +15 @@
 # tempfile for fitsdump -h output
 
-# options: 
-skipmd5sum="no" # fill md5 sums in any case
-skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
-skipmd5sum="yes" # do not fill md5 sums in any case #new default since 2012/06/05 when the checksum is available in heaser
+## options: 
+#skipmd5sum="no" # fill md5 sums in any case
+#skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
+#skipmd5sum="yes" # do not fill md5 sums in any case #new default since 2012/06/05 when the checksum is available in heaser
 
 doupdate="yes" # update all entries
 doupdate="no" # fill only entries which are not yet existing #default
 
+root=/opt/root_svn/bin/thisroot.sh
+
 source `dirname $0`/../Sourcefile.sh
-printprocesslog "INFO starting $0 with options doupdate="$doupdate" and skipmd5sum="$skipmd5sum
+printprocesslog "INFO starting $0 with options doupdate="$doupdate #" and skipmd5sum="$skipmd5sum
 
 # setup to use ftools
@@ -42 +44 @@
    finish
 fi
-if ! ls /loc_data/raw >/dev/null 2>&1
+if ! ls /newdaq/raw >/dev/null 2>&1
 then 
-   printprocesslog "ERROR /loc_data/raw is not available."
+   printprocesslog "ERROR /newdaq/raw is not available."
    finish
 fi
@@ -55 +57 @@
 # get last 3, 6 or 9 nights
 dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \
-#        `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \
+        `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \
 #        `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \
         )
@@ -68 +70 @@
    query="SELECT fNight FROM AuxFilesAvailISDCStatus WHERE fNight="$date2
    night=`sendquery`
-   if [ "$night" == "" ]
+   if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    then
       printprocesslog "INFO insert "$date2" to AuxFilesAvailISDCStatus"
@@ -77 +79 @@
    query="SELECT fNight FROM DriveFileAvailISDCStatus WHERE fNight="$date2
    night=`sendquery`
-   if [ "$night" == "" ]
+   if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    then
       printprocesslog "INFO insert "$date2" to DriveFileAvailISDCStatus"
@@ -86 +88 @@
    query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$date2
    night=`sendquery`
-   if [ "$night" == "" ]
+   if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1
    then
       printprocesslog "INFO insert "$date2" to AuxDataInsertStatus"
@@ -118 +120 @@
       # raw and original file
       # file: /loc_data/zipraw
-      # rawfile: /loc_data/raw
-      # origfile: /daq/raw
-      rawfile=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
-      origfile=`echo $rawfile | sed -e 's/loc_data/daq/'`
+      # rawfile: /daq/raw
+      # origfile: /newdaq/raw
+      rawfile=`echo $file | sed -e 's/loc_data/daq/' -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
+      rawfile2=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
+      origfile=`echo $rawfile | sed -e 's/daq/newdaq/'`
 
       # check if it is drs file
@@ -131 +134 @@
       fi
 
-      # check if file is already finished
-      # original file on daq (if data was taken on daq
-      if [ -e $origfile ]
-      then
-         # check if raw file was changed in the last 30 minutes
-         isnew=`find $origfile -cmin -30`
-         if [ "$isnew" != "" ]
-         then
-            printprocesslog "WARN "$origfile" is not older than 30 min. -> continue"
-            continue
-         fi
-
-         # get time of last modification as seconds since Epoch for both files
-         timeorig=`stat -c %Y $origfile`
-         timecopy=`stat -c %Y $rawfile`
-         # compare times
-         if ! [ $timeorig -eq $timecopy ]
-         then
-            # if times are not the same, the file is still open => no check
-            printprocesslog "INFO file "$rawfile" not yet closed. -> continue"
-            continue
-         fi
-      else
-         # if the origfile doesn't exist, the data was probably written not on daq but on data
-         printprocesslog "INFO file "$rawfile" was probably taken on data and not daq."
-      fi
+# not needed anymore as files are only rsynced from newdaq to daq when they are closed
+#      # check if file is already finished
+#      # original file on daq (if data was taken on daq
+#      if [ -e $origfile ]
+#      then
+#         # check if raw file was changed in the last 30 minutes
+#         isnew=`find $origfile -cmin -30`
+#         if [ "$isnew" != "" ]
+#         then
+#            printprocesslog "WARN "$origfile" is not older than 30 min. -> continue"
+#            continue
+#         fi
+#
+#         # get time of last modification as seconds since Epoch for both files
+#         timeorig=`stat -c %Y $origfile`
+#         timecopy=`stat -c %Y $rawfile`
+#         # compare times
+#         if ! [ $timeorig -eq $timecopy ]
+#         then
+#            # if times are not the same, the file is still open => no check
+#            printprocesslog "INFO file "$rawfile" not yet closed. -> continue"
+#            continue
+#         fi
+#      else
+#         # if the origfile doesn't exist, the data was probably written not on daq but on data
+#         printprocesslog "INFO file "$rawfile" was probably taken on data and not daq."
+#      fi
       
       # get run and file number form filename
@@ -189 +193 @@
       numfitserrors=0
       #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
-      numfitserrors=`fverify $rawfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
+      #numfitserrors=`fverify $rawfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
+# ftools missing on daq
+#      numfitserrors=`ssh daq "export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/ ; source $HEADAS/headas-init.sh ; fverify $rawfile2 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'"`
+#      echo $rawfile2" ------ "$numfitserrors
       #if [ "$checkfitsfile" == "" ]
       if [ $numfitserrors -gt 0 ]
       then
-         printprocesslog "WARN "$rawfile" has "$numfitserrors" fitserror(s). "
+         printprocesslog "WARN "$rawfile2" has "$numfitserrors" fitserror(s). "
          fitsdumperrors=`$factpath/fitsdump -h -t Events $file 2>&1 | grep corrupted`
          if [ "$fitsdumperrors" != "" ]
@@ -200 +207 @@
          fi
          #numfitserrors=1
+      fi
+
+      # check if file was closed properly
+      if [ "`echo $file | grep -o drs`" == "drs" ]
+      then
+         nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
+         tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
+      else
+         tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
+      fi
+      if [ "$tstop" == "0" ]
+      then
+         echo $file" not yet closed."
+         echo "WARN: $file has empty TSTOP"
+         filecorrupt="yes"
       fi
 
@@ -361 +383 @@
       fi
 
-      # check if entry has already checksums
-      query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
-      query=$query" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
-      printprocesslog "DEBUG check if md5sums are alreay in DB. QUERY:"$query
-      #result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"`
-      result5=`sendquery`
-
-      # get md5sums of raw and zip file
-      #   to safe time for tests and update this can be skipped ($skipmd5sum)
-      md5sum=
-      md5sumzip=
-      if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ]
-      then
-         #echo "skip: "$skipmd5sum >> $logfile 2>&1
-         #echo "res5: -"$result5"-" >> $logfile 2>&1
-         #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ] 
-         if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ] 
-         then 
-            #zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-            # only do the md5sum if the zipfile is already available
-            if ls $file >/dev/null 2>&1
-            then 
-               printprocesslog "INFO calculation md5sum for rawfile "$rawfile
-               md5sum=`md5sum $rawfile | cut -d' ' -f1`
-               printprocesslog "INFO calculation md5sum for zipfile "$file
-               md5sumzip=`md5sum $file | cut -d' ' -f1`
-            fi
-         fi
-      fi
+#      # check if entry has already checksums
+#      query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
+#      query=$query" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
+#      printprocesslog "DEBUG check if md5sums are alreay in DB. QUERY:"$query
+#      #result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"`
+#      result5=`sendquery`
+#
+#      # get md5sums of raw and zip file
+#      #   to safe time for tests and update this can be skipped ($skipmd5sum)
+#      md5sum=
+#      md5sumzip=
+#      if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ]
+#      then
+#         #echo "skip: "$skipmd5sum >> $logfile 2>&1
+#         #echo "res5: -"$result5"-" >> $logfile 2>&1
+#         #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ] 
+#         if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ] 
+#         then 
+#            #zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+#            # only do the md5sum if the zipfile is already available
+#            if ls $file >/dev/null 2>&1
+#            then 
+#               printprocesslog "INFO calculation md5sum for rawfile "$rawfile
+#               md5sum=`md5sum $rawfile | cut -d' ' -f1`
+#               printprocesslog "INFO calculation md5sum for zipfile "$file
+#               md5sumzip=`md5sum $file | cut -d' ' -f1`
+#            fi
+#         fi
+#      fi
 
       # insert or update depending on whether run exists
@@ -460 +482 @@
          query=$query", fDataSum='"$datasum"'"
       fi
-      if [ "$md5sum" != "" ]
-      then
-         query=$query", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
-      fi
+#      if [ "$md5sum" != "" ]
+#      then
+#         query=$query", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
+#      fi
       if [ "$numdrsfiles" != "" ]
       then