source: trunk/DataCheck/Transfer/ZipRawData.sh@ 17556

#!/bin/bash

# this script has been written to run on La Palma on the machine data
#   i.e. paths are only working on this machine

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting "$0

logfile=$runlogpath"/ZipRawLP-"$datetime".log"
date >> $logfile

if ! ls /daq/raw >/dev/null 2>&1
then 
   printprocesslog "ERROR /daq/raw is not available."
   finish
fi

# get dates
if [ "$certaindate" != "" ]
then
   getdates $certaindate
else
   # get all night
   #getdates "all"
   # get last 3 nights if hour between 7 and 19h, else only current night
   getdates 3 7 19
fi

# setup to use ftools
source $HEADAS/headas-init.sh

delaytime=5

# do rsync for rawfiles of these dates
for date in ${dates[@]}
do 
   echo "" >> $logfile 2>&1
   rawdir=/daq/raw/$date
   echo `date`": processing files in "$rawdir >> $logfile 2>&1
   printprocesslog "INFO processing files in "$rawdir
   #echo "INFO processing files in "$rawdir
   # check if data are available from that night
   if ! [ -d $rawdir ]
   then
      echo `date`": no data available in "$rawdir >> $logfile 2>&1
      printprocesslog "INFO no data available in "$rawdir
      continue
   fi
   zipdir=/loc_data/zipraw/$date
   if ! [ -d $zipdir ]
   then
      # create output directory for zip
      makedir $zipdir >> $logfile 2>&1
   fi

   # find all fits-files starting with the oldest file
   echo `date`": finding files to be zipped in $rawdir..." >> $logfile 2>&1
   printprocesslog "INFO finding files to be zipped in "$rawdir"..."
   fitsfiles=`find $rawdir -type f -name '*fits'| sort `

   if [ ${#fitsfiles[@]} -eq 0 ]
   then
      echo `date`": no files to be zipped in $rawdir..." >> $logfile 2>&1
      printprocesslog "INFO no files to be zipped in "$rawdir"..."
      continue
   fi

   # loop to zip files
   echo `date`": zipping files in $rawdir..." >> $logfile 2>&1
   printprocesslog "INFO zipping files in "$rawdir"..."
   for file in $fitsfiles
   do
      # filename for temporary and final zipfile
      zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
      zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
      # check if zipped file already exists
      if [ -e $zipfile ]
      then
         continue
      fi

      # check if raw file was accessed in the last $delaytime minutes
      isnew=`find $file -amin -$delaytime`
      if [ "$isnew" != "" ]
      then
         echo $file" is not older than $delaytime min => continue" >> $logfile 2>&1
         printprocesslog "INFO "$file" is not older than $delaytime min => continue"
         continue
      fi

# not needed anymore, as only complete files are transferred from newdaq to daq
#      # check if file is already finished
#      # original file on daq (if data was taken on daq
#      origfile=`echo $file | sed -e 's/loc_data/daq/'`
#      if [ -e $origfile ]
#      then
#         # get time of last modification as seconds since Epoch for both files
#         timeorig=`stat -c %Y $origfile`
#         timecopy=`stat -c %Y $file`
#         # compare times
#         if ! [ $timeorig -eq $timecopy ]
#         then
#            # if times are not the same, the file is still open => no zip
#            echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1
#            printprocesslog "WARN file "$file" not yet closed => continue"
#            continue
#         fi
#      else
#         # if the origfile doesn't exist, the data was probably written not on daq but on data
#         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1
#         printprocesslog "WARN file "$file" was probably taken on data and not daq "
#      fi

#      # update the raw file with the checksums
#      ftchecksum update=yes $file >> $logfile 2>&1
#      check=$?
#      if [ $check -eq 0 ]
#      then
#         echo `date`": file "$file" was updated with the checksums " >> $logfile 2>&1
#      else
#         echo `date`": problem when updating file "$file" (ftchecksum exited with "$check")" >> $logfile 2>&1
#         echo `date`": problem when updating file "$file" (ftchecksum exited with "$check")"
#         continue
#      fi

      echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile 2>&1
      printprocesslog "INFO zipping "$file" to "$zipfile" ..."
      #echo "INFO zipping "$file" to "$zipfile" ..."
      # read setup again to allow for updates of variables defining transfer
      source `dirname $0`/../Sourcefile.sh
      # zip file to stdout and pipe it to outputfile
      echo "pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp" >> $logfile 2>&1
      pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp
      statuspigz=( `echo ${PIPESTATUS[@]}` )
      printprocesslog "DEBUG PIPESTATUS:"${statuspigz[@]}
      echo " PIPESTATUS:"${statuspigz[@]} >> $logfile 2>&1
      if [ ${statuspigz[0]} -eq 0 ] && [ ${statuspigz[1]} -eq 0 ]
      then
         # if successful, move temporary to final zipfile
         printprocesslog "INFO move "$zipfiletmp" to "$zipfile" ..."
         mv -v $zipfiletmp $zipfile >> $logfile 2>&1
      else
         # if not successful, remove temporary zipfile
         printprocesslog "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")."
         echo "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")." >> $logfile 2>&1
         finish
         printprocesslog "INFO remove "$zipfiletmp"..."
         rm -v $zipfiletmp >> $logfile 2>&1
         diskusage=( `df -P /loc_data | grep /loc_data ` )
         # check if more than ~ 10 GB are left on /loc_data
         if [ ${diskusage[3]} -lt 10000000 ] 
         then
            echo "WARN less than 10 GB left on /loc_data on data " >> $logfile 2>&1
            printprocesslog "WARN less than 10 GB left on /loc_data on data "
            finish
         fi
      fi
   done
done
echo "finished zipping..." >> $logfile 2>&1
finish