source: trunk/DataCheck/ZipRawData.sh@ 12575

#!/bin/bash

year=`date +%Y --date="-1day"`
month=`date +%m --date="-1day"`

today=`date +%F`
logfile=/home/`whoami`/DataCheck/log/ZipRaw$today.log

echo "" >> $logfile 2>&1
echo "" >> $logfile 2>&1
echo "" >> $logfile 2>&1
rawdir=/loc_data/raw/$year/$month
echo `date`": processing files in "$rawdir >> $logfile

#find all directories in $rawdir
dirs=`find $rawdir -type d | sort`

echo `date`": create missing directories in /loc_data/zipraw/$year/$month" >> $logfile
for dir in $dirs 
do 
   zipdir=`echo $dir | sed -e 's/raw/zipraw/'` 
   # check if output directory for zip already exists
   if [ -d $zipdir ] 
   then 
      continue
   fi
   # create output directory for zip
   mkdir -pv $zipdir >> $logfile
done

# find all fits-files starting with the oldest file
echo `date`": finding files to be zipped in $rawdir..." >> $logfile
fitsfiles=`find $rawdir -type f -name '*fits'| sort `

# loop to zip files
echo `date`": zipping files in $rawdir..." >> $logfile
for file in $fitsfiles 
do 
   # filename for temporary and final zipfile
   zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` 
   zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'` 
   # check if zipped file already exists
   if [ -e $zipfile ] 
   then 
      continue
   fi

   # check if files was accessed in the last 30 minutes
   isnew=`find $file -amin -30`
   if [ "$isnew" != "" ]
   then
      echo $file" is not older than 30 min => continue" >> $logfile
      continue
   fi

   # check if file is already finished
   # original file on daq (if data was taken on daq
   origfile=`echo $file | sed -e 's/loc_data/daq/'` 
   if [ -e $origfile ]
   then
      # get time of last modification as seconds since Epoch for both files
      timeorig=`stat -c %Y $origfile`
      timecopy=`stat -c %Y $file`
      # compare times
      if ! [ $timeorig -eq $timecopy ]
      then
         # if times are not the same, the file is still open => no zip
         echo `date`": file "$file" not yet closed => continue" >> $logfile
         continue
      fi
   else
      # if the origfile doesn't exist, the data was probably written not on daq but on data
      echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
   fi

   echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile
   # zip file to stdout and pipe it to outputfile
   if pigz -1 -c -f $file > $zipfiletmp;  
   then
      # if successful, move temporary to final zipfile
      mv -v $zipfiletmp $zipfile >> $logfile
   else
      # if not successful, remove temporary zipfile
      rm -v $zipfiletmp >> $logfile
   fi
done