source: trunk/DataCheck/CheckRawData.sh@ 12607

#!/bin/bash

today=`date +%F`
logfile=/home/`whoami`/DataCheck/log/CheckRaw$today.log

password=`cat /home/fact/DataCheck/.pw`
if [ "$password" == "" ]
then
   echo "please insert password in .pw file"
fi

# setup to use ftools
export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
source $HEADAS/headas-init.sh

# get last 3 nights
dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
dates=( `date +%Y/%m/%d --date="-1day"` )

# do rsync for rawfiles of these dates
for date in ${dates[@]}
do 
   echo "" >> $logfile 2>&1
   echo "" >> $logfile 2>&1
   echo "" >> $logfile 2>&1
   rawdir=/loc_data/raw/$date
   runnumber=`echo $date | sed -e 's/\///g'`
   echo `date`": processing files in "$rawdir >> $logfile
   # check if data are available from that night
   if ! [ -d $rawdir ]
   then
      echo `date`": no data available in "$rawdir >> $logfile
      continue
   fi

   # find all fits-files starting with the oldest file
   echo `date`": finding files to be zipped in $rawdir..." >> $logfile
   fitsfiles=`find $rawdir -type f -name '*.fits'| sort `

   # loop to zip files
   echo `date`": zipping files in $rawdir..." >> $logfile
   for file in $fitsfiles
   do
      # check if raw file was accessed in the last 30 minutes
      isnew=`find $file -amin -30`
      if [ "$isnew" != "" ]
      then
         echo $file" is not older than 30 min => continue" >> $logfile
         continue
      fi

      # check if it is drs file
      isnew=`ls $file | grep drs`
      if [ "$isnew" != "" ]
      then
         echo $file" is a drs file => continue" >> $logfile
         continue
      fi

      # check if file is already finished
      # original file on daq (if data was taken on daq
      origfile=`echo $file | sed -e 's/loc_data/daq/'`
      if [ -e $origfile ]
      then
         # get time of last modification as seconds since Epoch for both files
         timeorig=`stat -c %Y $origfile`
         timecopy=`stat -c %Y $file`
         # compare times
         if ! [ $timeorig -eq $timecopy ]
         then
            # if times are not the same, the file is still open => no zip
            echo `date`": file "$file" not yet closed => continue" >> $logfile
            continue
         fi
      else
         # if the origfile doesn't exist, the data was probably written not on daq but on data
         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
      fi
      
      # get md5sum
      md5sum=`md5sum $file | cut -d' ' -f1`
      zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
      md5sumzip=`md5sum $zipfile | cut -d' ' -f1`
      
      # get and check run and file number
      runnumbererror="no"
      numbererror="no"
      numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
      runnumberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
      runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
      filenumberfromfileorig=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
      filenumberfromfile=`printf %03d $filenumberfromfileorig`
      numberfromfile=$runnumberfromfile"_"$filenumberfromfile
      if [ "$runnumber" != "$runnumberfromname" ]
      then
         runnumbererror="yes"
      fi
      if [ "$numberfromfile" != "$numberfromname" ]
      then
         numbererror="yes"
      fi
      
      # get other variables from header 
      runtype=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
      roi=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
      numevents=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`

      # check if fits file is corrupted
      fitsfileerror="no"
      checkfitsfile=`fverify $file | grep '0 error(s)'`
      if [ "$checkfitsfile" == "" ]
      then
         fitsfileerror="yes"
      fi
      echo "summary for file"$file
      echo "  errors: "$fitsfileerror
      echo "  runnumber error: "$runnumbererror
      echo "  number error: "$numbererror
      echo "  roi: "$roi
      echo "  runtype: "$runtype
      echo "  numevents: "$numevents
      echo "  md5sum: "$md5sum
      echo "  md5sum(zip): "$md5sumzip
      # check wehter entry has been made (status table)
      # get runtype
      query2="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
      echo "Q2:"$query2
      result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` )
      if [ ${#result2} -eq 0 ]
      then 
         echo "Could not query fRunTypeKey for runtype "$runtype
         exit
      fi
      # check if entry already exists
      query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
      echo "Q3:"$query3
      resul3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
      if [ "$result3" == "" ]
      then 
         query4="INSERT"
         querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromfileorig", "
      else
         query4="UPDATE"
         querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
      fi
      query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}", fNumEvents="$numevents", fROI="$roi
      query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
      echo "Q4:"$query4
      if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query4"
      then
         echo "insert/update of "$numfromfile" to mysql"
      fi
      exit
   done
done