source: trunk/DataCheck/ZipRawData.sh@ 12607

#!/bin/bash

today=`date +%F`
logfile=/home/`whoami`/DataCheck/log/ZipRaw$today.log

# get last 3 nights
dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )

# do rsync for rawfiles of these dates
for date in ${dates[@]}
do 
   echo "" >> $logfile 2>&1
   echo "" >> $logfile 2>&1
   echo "" >> $logfile 2>&1
   rawdir=/loc_data/raw/$date
   echo `date`": processing files in "$rawdir >> $logfile
   # check if data are available from that night
   if ! [ -d $rawdir ]
   then
      echo `date`": no data available in "$rawdir >> $logfile
      continue
   fi
   zipdir=/loc_data/zipraw/$date
   if ! [ -d $zipdir ]
   then
      # create output directory for zip
      mkdir -pv $zipdir >> $logfile
   fi

   # find all fits-files starting with the oldest file
   echo `date`": finding files to be zipped in $rawdir..." >> $logfile
   fitsfiles=`find $rawdir -type f -name '*fits'| sort `

   # loop to zip files
   echo `date`": zipping files in $rawdir..." >> $logfile
   for file in $fitsfiles
   do
      # filename for temporary and final zipfile
      zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
      zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
      # check if zipped file already exists
      if [ -e $zipfile ]
      then
         continue
      fi

      # check if raw file was accessed in the last 30 minutes
      isnew=`find $file -amin -30`
      if [ "$isnew" != "" ]
      then
         echo $file" is not older than 30 min => continue" >> $logfile
         continue
      fi

      # check if file is already finished
      # original file on daq (if data was taken on daq
      origfile=`echo $file | sed -e 's/loc_data/daq/'`
      if [ -e $origfile ]
      then
         # get time of last modification as seconds since Epoch for both files
         timeorig=`stat -c %Y $origfile`
         timecopy=`stat -c %Y $file`
         # compare times
         if ! [ $timeorig -eq $timecopy ]
         then
            # if times are not the same, the file is still open => no zip
            echo `date`": file "$file" not yet closed => continue" >> $logfile
            continue
         fi
      else
         # if the origfile doesn't exist, the data was probably written not on daq but on data
         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
      fi

      echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile
      # zip file to stdout and pipe it to outputfile
      if pigz -1 -c -f $file > $zipfiletmp;
      then
         # if successful, move temporary to final zipfile
         mv -v $zipfiletmp $zipfile >> $logfile
      else
         # if not successful, remove temporary zipfile
         rm -v $zipfiletmp >> $logfile
      fi
   done
done