#!/bin/bash

# this script has been written to run on La Palma on the machine newdata
#   i.e. paths are only working on this machine

# to use script with /data1 instead of /scratch 
#  (e.g. when /scratch is full) 
# export data1=yes
# before executing the script
source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting "$0

logfile=$runlogpath"/ZipRawLP-"$datetime".log"
date >> $logfile

if ! ls /scratch/raw >/dev/null 2>&1
then 
   printprocesslog "ERROR /scratch/raw is not available."
   finish
fi

# get dates
if [ "$certaindate" != "" ]
then
   getdates $certaindate
else
   # get all night
   #getdates "all"
   # get last 3 nights if hour between 7 and 19h, else only current night
   getdates 3 7 19
fi

rsynctempdir=/data1/rsync_tmp
if ! [ -d $rsynctempdir ]
then
   mkdir $rsynctempdir
fi

# not needed anymore as ftools are not used anymore
## setup to use ftools
#source $HEADAS/headas-init.sh

# files younger than $delaytime minutes are not processed
delaytime=5

# do rsync for rawfiles of these dates
for date in ${dates[@]}
do 
   echo "" >> $logfile 2>&1
   if [ "$data1" = "yes" ]
   then 
      rawdir=/data1/raw/$date
      printprocesslog "INFO using "$rawdir" as input"
   else 
      rawdir=/scratch/raw/$date
   fi
   echo `date`": processing files in "$rawdir >> $logfile 2>&1
   printprocesslog "INFO processing files in "$rawdir
   #echo "INFO processing files in "$rawdir
   # check if data are available from that night
   if ! [ -d $rawdir ]
   then
      echo `date`": no data available in "$rawdir >> $logfile 2>&1
      printprocesslog "INFO no data available in "$rawdir
      continue
   fi

   # find all fits-files starting with the oldest file
   echo `date`": finding files to be zipped in $rawdir..." >> $logfile 2>&1
   printprocesslog "INFO finding files to be zipped in "$rawdir"..."
   fitsfiles=`find $rawdir -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort `

   if [ ${#fitsfiles[@]} -eq 0 ]
   then
      echo `date`": no files to be zipped in $rawdir..." >> $logfile 2>&1
      printprocesslog "INFO no files to be zipped in "$rawdir"..."
      continue
   fi
   
   rawdir2=/data1/raw/$date
   zipdir=/data2/zipraw/$date
   # create output directories
   makedir $rawdir2 >> $logfile 2>&1
   makedir $zipdir >> $logfile 2>&1

   # loop to zip files
   echo `date`": zipping files in $rawdir..." >> $logfile 2>&1
   printprocesslog "INFO zipping files in "$rawdir"..."
   for file in $fitsfiles
   do
      # check if more than ~ 10 GB are left on output directories
      diskusage=( `df -P /data1 | grep /data1 ` )
      if [ ${diskusage[3]} -lt 10000000 ] 
      then
         echo "WARN less than 10 GB left on /data1 on newdata ("${diskusage[3]}")" >> $logfile 2>&1
         printprocesslog "WARN less than 10 GB left on /data1 on newdata ("${diskusage[3]}")"
         finish
      fi
      diskusage=( `df -P /data2 | grep /data2 ` )
      if [ ${diskusage[3]} -lt 10000000 ] 
      then
         echo "WARN less than 10 GB left on /data2 on newdata ("${diskusage[3]}")" >> $logfile 2>&1
         printprocesslog "WARN less than 10 GB left on /data2 on newdata ("${diskusage[3]}")"
         finish
      fi
      
      # check if raw file was accessed in the last $delaytime minutes
      isnew=`find $file -amin -$delaytime`
      if [ "$isnew" != "" ]
      then
         echo $file" is not older than $delaytime min => continue" >> $logfile 2>&1
         printprocesslog "INFO "$file" is not older than $delaytime min => continue"
         continue
      fi
      
      printprocesslog "INFO process file "$file
      if [ "$certaindate" != "" ]
      then
         echo "INFO process file "$file
      fi
      # first rsync from /scratch to /data1
      if [ "$data1" != "yes" ]
      then 
         file2=`echo $file | sed -e 's/scratch/data1/'`
         if ! rsync -au -T $rsynctempdir $file $file2
         then
            printprocesslog "ERROR something went wrong with rsync of "$file
            rm $file2
            continue
         fi
      fi
      # copying to /data2
      iszipped=`echo $file | grep -o fz`
      if [ "$iszipped" == "fz" ]
      then
         # treat already compressed files
         printprocesslog "DEBUG treat compressed file "$file

         # filename for file on data
         if [ "$data1" = "yes" ]
         then 
            zipfile=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' `
            zipfiletmp=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
         else
            zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' `
            zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
         fi
         # check if file on data already exists
         if [ -e $zipfile ]
         then
            continue
         fi

         echo `date`": copying "$file" to "$zipfile" ..." >> $logfile 2>&1
         printprocesslog "INFO copying "$file" to "$zipfile" ..."
         #echo "INFO zipping "$file" to "$zipfile" ..."
         # read setup again to allow for updates of variables defining transfer
         source `dirname $0`/../Sourcefile.sh
         # zip file to stdout and pipe it to outputfile
         echo "pv --rate-limit $limitpigz $file > $zipfiletmp" >> $logfile 2>&1
         pv --rate-limit $limitpigz $file > $zipfiletmp
         statuspv=$?
         printprocesslog "DEBUG pvstatus:"$statuspv
         echo " pvstatus:"$statuspv >> $logfile 2>&1
         if [ $statuspv -eq 0 ] 
         then
            # if successful, move temporary to final zipfile
            printprocesslog "INFO move "$zipfiletmp" to "$zipfile" ..."
            mv -v $zipfiletmp $zipfile >> $logfile 2>&1
         else
            # if not successful, remove temporary zipfile
            printprocesslog "WARN pv failed (statuspv:"$statuspv")."
            echo "WARN pv failed (PIPESTATUS:"$statuspv")." >> $logfile 2>&1
            printprocesslog "INFO remove "$zipfiletmp"..."
            rm -v $zipfiletmp >> $logfile 2>&1
            finish
         fi
      else
         # treat uncompressed files
         printprocesslog "DEBUG treat uncompressed file "$file

         # filename for temporary and final zipfile
         if [ "$data1" = "yes" ]
         then 
            zipfile=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
            zipfiletmp=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
         else
            zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
            zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
         fi
         
         # check if zipped file already exists
         if [ -e $zipfile ]
         then
            continue
         fi

         echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile 2>&1
         printprocesslog "INFO zipping "$file" to "$zipfile" ..."
         #echo "INFO zipping "$file" to "$zipfile" ..."
         # read setup again to allow for updates of variables defining transfer
         source `dirname $0`/../Sourcefile.sh
         # zip file to stdout and pipe it to outputfile
         echo "pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp" >> $logfile 2>&1
         pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp
         statuspigz=( `echo ${PIPESTATUS[@]}` )
         printprocesslog "DEBUG PIPESTATUS:"${statuspigz[@]}
         echo " PIPESTATUS:"${statuspigz[@]} >> $logfile 2>&1
         if [ ${statuspigz[0]} -eq 0 ] && [ ${statuspigz[1]} -eq 0 ]
         then
            # if successful, move temporary to final zipfile
            printprocesslog "INFO move "$zipfiletmp" to "$zipfile" ..."
            mv -v $zipfiletmp $zipfile >> $logfile 2>&1
         else
            # if not successful, remove temporary zipfile
            printprocesslog "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")."
            echo "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")." >> $logfile 2>&1
            printprocesslog "INFO remove "$zipfiletmp"..."
            rm -v $zipfiletmp >> $logfile 2>&1
            finish
         fi
      fi
   done
done
echo "finished zipping..." >> $logfile 2>&1
finish