source: trunk/DataCheck/CheckAuxData.sh@ 12706

#!/bin/bash

# 
# some notes: 
# 
# get threshold (assuming that the threshold is the same for all patches and the last value is the highest)
# fstatistic /loc_data/aux/2011/11/25/20111125_042.FTM_CONTROL_STATIC_DATA.fits[1] col=PatchThresh[1] rows=-

today=`date +%F`
logfile=/home/`whoami`/DataCheck/log/CheckRaw$today.log

# setup to use ftools
export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
source $HEADAS/headas-init.sh

# get last 3 nights
dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
dates=( `date +%Y/%m/%d` )

# do rsync for rawfiles of these dates
for date in ${dates[@]}
do 
   echo "" >> $logfile 2>&1
   echo "" >> $logfile 2>&1
   echo "" >> $logfile 2>&1
   rawdir=/loc_data/aux/$date
   runnumber=`echo $date | sed -e 's/\///g'`
   echo `date`": processing files in "$rawdir >> $logfile
   # check if data are available from that night
   if ! [ -d $rawdir ]
   then
      echo `date`": no data available in "$rawdir >> $logfile
      continue
   fi

   # find all fits-files starting with the oldest file
   echo `date`": finding files to be zipped in $rawdir..." >> $logfile
   fitsfiles=`find $rawdir -type f -name '*fits'| sort `

   # loop to zip files
   echo `date`": zipping files in $rawdir..." >> $logfile
   for file in $fitsfiles
   do
      # check if raw file was accessed in the last 30 minutes
      isnew=`find $file -amin -5`
      if [ "$isnew" != "" ]
      then
         echo $file" is not older than 30 min => continue" >> $logfile
         continue
      fi

      # check if file is already finished
      # original file on daq (if data was taken on daq
      origfile=`echo $file | sed -e 's/loc_data/daq/'`
      if [ -e $origfile ]
      then
         # get time of last modification as seconds since Epoch for both files
         timeorig=`stat -c %Y $origfile`
         timecopy=`stat -c %Y $file`
         # compare times
         if ! [ $timeorig -eq $timecopy ]
         then
            # if times are not the same, the file is still open => no zip
            echo `date`": file "$file" not yet closed => continue" >> $logfile
            continue
         fi
      else
         # if the origfile doesn't exist, the data was probably written not on daq but on data
         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
      fi
      
      # md5sum -> db
      # runtype
      # position/source name
      # # evts
      # trigger setup
      # roi
      # check run#
      
      # both
      md5sum=`md5sum $file | cut -d' ' -f1`
      numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
      runnumberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`# + compare to $runnumber (=$date)
      # aux file
      tablename=`echo $file | grep -E -o '[.][A-Z_]+[.]' | sed -e 's/[.]//g'`
      # raw file
      runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
      filenumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
      numberfromfile=$runnumberfromfile"_"$filenumberfromfile
      if [ "$numberfromfile" == "$numberfromname" ]
      then
         echo "numbers are identical"
      fi
      runtype=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
      roi=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
      numevents=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`

      # both
      checkfitsfile=`fverify $file | grep '0 error(s)'`
      if [ "$checkfitsfile" == "" ]
      then
         echo " "
         echo $file
         echo "file "$file" has probably an error"
      fi
      exit
   done
done