source: trunk/DataCheck/Transfer/RsyncRawLP.sh@ 14905

#!/bin/bash

# this script has been written to run on La Palma on the machine data
#   i.e. paths are only working on this machine

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting "$0

logfile=$runlogpath"/RsyncRawLP-"$datetime".log"
date >> $logfile

# check if /daq is mounted on data
if ! mount | grep daq >> $logfile 2>&1
then 
   printprocesslog "ERROR /daq is not mounted on data => please mount it"
   echo `date`": /daq is not mounted on data => please mount it"
   finish
fi

# check if paths are available
if ! ls /daq/raw >/dev/null 2>&1
then 
   printprocesslog "ERROR /daq/raw is not available."
   finish
fi
if ! ls /loc_data/raw >/dev/null 2>&1
then 
   printprocesslog "ERROR /loc_data/raw is not available."
   finish
fi

# get last 3, 6 or 9 nights
#dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \
#        `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \
#        `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \
#        )

dates=( 
        `date +%Y/%m/%d --date="-204hour"`  `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-156hour"` \
        `date +%Y/%m/%d --date="-132hour"`  `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-84hour"`  \
        `date +%Y/%m/%d --date="-60hour"`   `date +%Y/%m/%d --date="-36hour"`  `date +%Y/%m/%d --date="-12hour"`  \
        )

# do rsync for rawfiles of these dates
for date in ${dates[@]}
do 
   echo "" >> $logfile 2>&1
   # raw data directories on data
   rawdirdaq=/daq/raw/$date/ # /loc_data from daq via nfs on data
   rawdirdata=/loc_data/raw/$date
   if ! [ -d $rawdirdaq ]
   then 
      echo `date`": "$rawdirdaq" not found. No data available on daq." >> $logfile 2>&1
      if [ -d $rawdirdata ]
      then
         printprocesslog "WARN "$rawdirdaq" not found, but "$rawdirdata". Data probably taken on data." >> $logfile 2>&1
      else
         printprocesslog "INFO "$rawdirdaq" not found. No data available on daq." >> $logfile 2>&1
      fi
      continue
   fi
   if ! [ -d $rawdirdata ]
   then 
      mkdir -pv $rawdirdata >> $logfile 2>&1
   fi
   printprocesslog "INFO processing files in "$rawdirdaq >> $logfile 2>&1
   echo `date`": processing files in "$rawdirdaq >> $logfile 2>&1

   # get current hour
   hour=`date +%k`
   # define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync
   if [ $hour -le 6 ] || [ $hour -ge 19 ]
   then
      # limit bw for rsync to 20 MB/s during night
      bwlimit="--bwlimit=20000"
      printprocesslog "INFO rsync data with "$bwlimit >> $logfile 2>&1
      echo "rsync data with "$bwlimit >> $logfile 2>&1
   else
      # no bw limit during day
      printprocesslog "INFO rsync data without bwlimit" >> $logfile 2>&1
      echo "rsync data without bwlimit" >> $logfile 2>&1
   fi

   #rsync from daq to data
   #if ! /usr/bin/rsync -avxHPu $bwlimit $rawdirdaq $rawdirdata >> $logfile 2>&1 # in case of adding checksum, only update files with rsync
   if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit $rawdirdaq $rawdirdata >> $logfile 2>&1
   then
      printprocesslog "WARN problem rsyncing rawdata for "$date" from daq to data"
      echo `date`": problem rsyncing rawdata for "$date" from daq to data"
   fi
done

finish