Index: trunk/DataCheck/CheckRawData.sh
===================================================================
--- trunk/DataCheck/CheckRawData.sh	(revision 12583)
+++ trunk/DataCheck/CheckRawData.sh	(revision 12583)
@@ -0,0 +1,157 @@
+#!/bin/bash
+
+today=`date +%F`
+logfile=/home/`whoami`/DataCheck/log/CheckRaw$today.log
+
+password=`cat /home/fact/DataCheck/.pw`
+if [ "$password" == "" ]
+then
+   echo "please insert password in .pw file"
+fi
+
+# setup to use ftools
+export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
+source $HEADAS/headas-init.sh
+
+# get last 3 nights
+dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
+dates=( `date +%Y/%m/%d --date="-1day"` )
+
+# do rsync for rawfiles of these dates
+for date in ${dates[@]}
+do 
+   echo "" >> $logfile 2>&1
+   echo "" >> $logfile 2>&1
+   echo "" >> $logfile 2>&1
+   rawdir=/loc_data/raw/$date
+   runnumber=`echo $date | sed -e 's/\///g'`
+   echo `date`": processing files in "$rawdir >> $logfile
+   # check if data are available from that night
+   if ! [ -d $rawdir ]
+   then
+      echo `date`": no data available in "$rawdir >> $logfile
+      continue
+   fi
+
+   # find all fits-files starting with the oldest file
+   echo `date`": finding files to be zipped in $rawdir..." >> $logfile
+   fitsfiles=`find $rawdir -type f -name '*.fits'| sort `
+
+   # loop to zip files
+   echo `date`": zipping files in $rawdir..." >> $logfile
+   for file in $fitsfiles
+   do
+      # check if raw file was accessed in the last 30 minutes
+      isnew=`find $file -amin -30`
+      if [ "$isnew" != "" ]
+      then
+         echo $file" is not older than 30 min => continue" >> $logfile
+         continue
+      fi
+
+      # check if it is drs file
+      isnew=`ls $file | grep drs`
+      if [ "$isnew" != "" ]
+      then
+         echo $file" is a drs file => continue" >> $logfile
+         continue
+      fi
+
+      # check if file is already finished
+      # original file on daq (if data was taken on daq
+      origfile=`echo $file | sed -e 's/loc_data/daq/'`
+      if [ -e $origfile ]
+      then
+         # get time of last modification as seconds since Epoch for both files
+         timeorig=`stat -c %Y $origfile`
+         timecopy=`stat -c %Y $file`
+         # compare times
+         if ! [ $timeorig -eq $timecopy ]
+         then
+            # if times are not the same, the file is still open => no zip
+            echo `date`": file "$file" not yet closed => continue" >> $logfile
+            continue
+         fi
+      else
+         # if the origfile doesn't exist, the data was probably written not on daq but on data
+         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
+      fi
+      
+      # get md5sum
+      md5sum=`md5sum $file | cut -d' ' -f1`
+      zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+      md5sumzip=`md5sum $zipfile | cut -d' ' -f1`
+      
+      # get and check run and file number
+      runnumbererror="no"
+      numbererror="no"
+      numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
+      runnumberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
+      runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
+      filenumberfromfileorig=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
+      filenumberfromfile=`printf %03d $filenumberfromfileorig`
+      numberfromfile=$runnumberfromfile"_"$filenumberfromfile
+      if [ "$runnumber" != "$runnumberfromname" ]
+      then
+         runnumbererror="yes"
+      fi
+      if [ "$numberfromfile" != "$numberfromname" ]
+      then
+         numbererror="yes"
+      fi
+      
+      # get other variables from header 
+      runtype=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
+      roi=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
+      numevents=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
+
+      # check if fits file is corrupted
+      fitsfileerror="no"
+      checkfitsfile=`fverify $file | grep '0 error(s)'`
+      if [ "$checkfitsfile" == "" ]
+      then
+         fitsfileerror="yes"
+      fi
+      echo "summary for file"$file
+      echo "  errors: "$fitsfileerror
+      echo "  runnumber error: "$runnumbererror
+      echo "  number error: "$numbererror
+      echo "  roi: "$roi
+      echo "  runtype: "$runtype
+      echo "  numevents: "$numevents
+      echo "  md5sum: "$md5sum
+      echo "  md5sum(zip): "$md5sumzip
+      # check wehter entry has been made (status table)
+      # get runtype
+      query2="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
+      echo "Q2:"$query2
+      result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` )
+      if [ ${#result2} -eq 0 ]
+      then 
+         echo "Could not query fRunTypeKey for runtype "$runtype
+         exit
+      fi
+      # check if entry already exists
+      query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
+      echo "Q3:"$query3
+      resul3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
+      if [ "$result3" == "" ]
+      then 
+         query4="INSERT"
+         querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromfileorig", "
+      else
+         query4="UPDATE"
+         querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
+      fi
+      query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}", fNumEvents="$numevents", fROI="$roi
+      query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
+      echo "Q4:"$query4
+      if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query4"
+      then
+         echo "insert/update of "$numfromfile" to mysql"
+      fi
+      exit
+   done
+done
+
+