Index: trunk/DataCheck/CheckRawData.sh
===================================================================
--- trunk/DataCheck/CheckRawData.sh	(revision 12608)
+++ trunk/DataCheck/CheckRawData.sh	(revision 12609)
@@ -3,4 +3,18 @@
 today=`date +%F`
 logfile=/home/`whoami`/DataCheck/log/CheckRaw$today.log
+
+skipmd5sum="no" # fill md5 sums in any case
+skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db
+#skipmd5sum="yes" # do not fill md5 sums in any case
+
+doupdate="yes" # update all entries
+#doupdate="no" # fill only entries which are not yet existing
+
+echo ""  >> $logfile 2>&1
+echo ""  >> $logfile 2>&1
+echo ""  >> $logfile 2>&1
+echo `date`"executing "$0"..." >> $logfile 2>&1
+echo "=====> doupdate: "$doupdate >> $logfile 2>&1
+echo "=====> skipmd5sum: "$skipmd5sum >> $logfile 2>&1
 
 password=`cat /home/fact/DataCheck/.pw`
@@ -16,5 +30,5 @@
 # get last 3 nights
 dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
-dates=( `date +%Y/%m/%d --date="-1day"` )
+dates=( `date +%Y/%m/%d --date="-2day"` )
 
 # do rsync for rawfiles of these dates
@@ -26,25 +40,27 @@
    rawdir=/loc_data/raw/$date
    runnumber=`echo $date | sed -e 's/\///g'`
-   echo `date`": processing files in "$rawdir >> $logfile
+   echo `date`": processing files in "$rawdir >> $logfile 2>&1
    # check if data are available from that night
    if ! [ -d $rawdir ]
    then
-      echo `date`": no data available in "$rawdir >> $logfile
+      echo `date`": no data available in "$rawdir >> $logfile 2>&1
       continue
    fi
 
    # find all fits-files starting with the oldest file
-   echo `date`": finding files to be zipped in $rawdir..." >> $logfile
+   echo `date`": finding files to be zipped in $rawdir..." >> $logfile 2>&1
    fitsfiles=`find $rawdir -type f -name '*.fits'| sort `
 
    # loop to zip files
-   echo `date`": zipping files in $rawdir..." >> $logfile
+   echo `date`": checking files in $rawdir..." >> $logfile 2>&1
    for file in $fitsfiles
    do
-      # check if raw file was accessed in the last 30 minutes
-      isnew=`find $file -amin -30`
+      echo "" >> $logfile 2>&1
+      echo "checking file "$file >> $logfile 2>&1
+      # check if raw file was changed in the last 30 minutes
+      isnew=`find $file -cmin -30`
       if [ "$isnew" != "" ]
       then
-         echo $file" is not older than 30 min => continue" >> $logfile
+         echo $file" is not older than 30 min => continue" >> $logfile 2>&1
          continue
       fi
@@ -54,5 +70,5 @@
       if [ "$isnew" != "" ]
       then
-         echo $file" is a drs file => continue" >> $logfile
+         echo $file" is a drs file => continue" >> $logfile 2>&1
          continue
       fi
@@ -70,16 +86,11 @@
          then
             # if times are not the same, the file is still open => no zip
-            echo `date`": file "$file" not yet closed => continue" >> $logfile
+            echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1
             continue
          fi
       else
          # if the origfile doesn't exist, the data was probably written not on daq but on data
-         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
-      fi
-      
-      # get md5sum
-      md5sum=`md5sum $file | cut -d' ' -f1`
-      zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-      md5sumzip=`md5sum $zipfile | cut -d' ' -f1`
+         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1
+      fi
       
       # get and check run and file number
@@ -101,43 +112,85 @@
       fi
       
+      # check if entry already exists
+      query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
+      echo "Q3:"$query3 >> $logfile 2>&1 2>&1
+      result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
+
+      # only proceed with file 
+      #  if information is not yet in database
+      #  and no update is wished ($doupdate)
+      if [ "$result3" != "" ] && [ "$doupdate" == "no" ]
+      then 
+         echo $file" has been inserted already => continue " >> $logfile 2>&1
+         continue
+      fi
+      
+      # check if entry has already checksums
+      query5="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
+      query5=$query5" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
+      echo "Q5:"$query5 >> $logfile 2>&1 2>&1
+      result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"`
+
+      # get md5sums of raw and zip file
+      #   to safe time for tests and update this can be skipped ($skipmd5sum)
+      md5sum=
+      md5sumzip=
+      if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ]
+      then
+         #echo "skip: "$skipmd5sum >> $logfile 2>&1
+         #echo "res5: -"$result5"-" >> $logfile 2>&1
+         #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ] 
+         if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ] 
+         then 
+            echo "calculation md5sum for raw..." >> $logfile 2>&1
+            md5sum=`md5sum $file | cut -d' ' -f1`
+            zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+            echo "calculation md5sum for zipraw..." >> $logfile 2>&1
+            md5sumzip=`md5sum $zipfile | cut -d' ' -f1`
+         fi
+      fi
+      
+      # check if this run has drs file
+      drsfile=`echo $file | sed -e 's/fits/drs.fits/'`
+      numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
+      
       # get other variables from header 
       runtype=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
       roi=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
       numevents=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
+      runstart=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
+      runstop=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
+      if [ "$runstart" == ""  ]
+      then
+         runstart=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
+      fi
+      if [ "$runstop" == ""  ]
+      then
+         runstop=`stat $file  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
+      fi
 
       # check if fits file is corrupted
-      fitsfileerror="no"
+      numfitserrors=0
       checkfitsfile=`fverify $file | grep '0 error(s)'`
       if [ "$checkfitsfile" == "" ]
       then
-         fitsfileerror="yes"
-      fi
-      echo "summary for file"$file
-      echo "  errors: "$fitsfileerror
-      echo "  runnumber error: "$runnumbererror
-      echo "  number error: "$numbererror
-      echo "  roi: "$roi
-      echo "  runtype: "$runtype
-      echo "  numevents: "$numevents
-      echo "  md5sum: "$md5sum
-      echo "  md5sum(zip): "$md5sumzip
-      # check wehter entry has been made (status table)
+         numfitserrors=1
+      fi
       # get runtype
       query2="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
-      echo "Q2:"$query2
+      echo "Q2:"$query2 >> $logfile 2>&1
       result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` )
       if [ ${#result2} -eq 0 ]
       then 
          echo "Could not query fRunTypeKey for runtype "$runtype
+         echo "Could not query fRunTypeKey for runtype "$runtype >> $logfile 2>&1
          exit
       fi
-      # check if entry already exists
-      query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
-      echo "Q3:"$query3
-      resul3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
+      # insert or update depending on whether run exists
       if [ "$result3" == "" ]
       then 
          query4="INSERT"
          querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromfileorig", "
+         querystop=""
       else
          query4="UPDATE"
@@ -145,11 +198,36 @@
       fi
       query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}", fNumEvents="$numevents", fROI="$roi
-      query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
-      echo "Q4:"$query4
+      if [ "$md5sum" != "" ]
+      then
+         query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
+      fi
+      query4=$query4", fHasDrsFile="$numdrsfiles
+      query4=$query4", fFitsFileErrors="$numfitserrors
+      query4=$query4", fRunStart='"$runstart"', fRunStop='"$runstop"'"
+      query4=$query4" "$querystop
+      echo "Q4:"$query4 >> $logfile 2>&1
       if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query4"
       then
-         echo "insert/update of "$numfromfile" to mysql"
-      fi
-      exit
+         echo "insert/update of "$numfromfile" to mysql" >> $logfile 2>&1
+      fi
+
+      # print summary for debug
+      echo "*******"  >> $logfile 2>&1
+      echo "summary for file "$file >> $logfile 2>&1
+      echo "*******"  >> $logfile 2>&1
+      echo "  errors: "$numfitserrors" (1 means errors exist)" >> $logfile 2>&1
+      echo "  number of drs files: "$numdrsfiles >> $logfile 2>&1
+      echo "  runnumber error: "$runnumbererror >> $logfile 2>&1
+      echo "  number error: "$numbererror >> $logfile 2>&1
+      echo "  roi: "$roi >> $logfile 2>&1
+      echo "  runtype: "$runtype >> $logfile 2>&1
+      echo "  numevents: "$numevents >> $logfile 2>&1
+      echo "  md5sum: "$md5sum >> $logfile 2>&1
+      echo "  md5sum(zip): "$md5sumzip >> $logfile 2>&1
+      echo "  start: "$runstart >> $logfile 2>&1
+      echo "  stop: "$runstop >> $logfile 2>&1
+
+      # missing
+      # check wether entry has been made (status table)
    done
 done
