source: trunk/DataCheck/ZipRawData.sh@ 12547

#!/bin/bash

year=`date +%Y --date="-1day"`
month=`date +%m --date="-1day"`

today=`date +%F`
logfile=/home/`whoami`/DataCheck/log/ZipRaw$today.log

rawdir=/loc_data/raw/$year/$month
echo `date`": processing files in "$rawdir >> $logfile

#find all directories in $rawdir
dirs=`find $rawdir -type d | sort`

echo `date`": create missing directories in /loc_data/zipraw/$year/$month" >> $logfile
for dir in $dirs 
do 
   zipdir=`echo $dir | sed -e 's/raw/zipraw/'` 
   # check if output directory for zip already exists
   if [ -d $zipdir ] 
   then 
      continue
   fi
   # create output directory for zip
   mkdir -pv $zipdir >> $logfile
done

echo `date`": finding files to be zipped in $rawdir..." >> $logfile
fitsfiles=`find $rawdir -type f -name '*fits' | sort -r`

echo `date`": zipping files in $rawdir..." >> $logfile
for file in $fitsfiles 
do 
   zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` 
   zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'` 
   # check if zipped file already exists
   if [ -e $zipfile ] 
   then 
      continue
   fi
   echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile
   # zip file to stdout and pipe it to outputfile
   if pigz -1 -c -f $file > $zipfiletmp;  
   then
      mv -v $zipfiletmp $zipfile >> $logfile
   else
      rm -v $zipfiletmp >> $logfile
   fi
done