source: trunk/DataCheck/Transfer/RsyncRawToISDC.sh@ 19610

#!/bin/bash
#

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"
program=RsyncRawToISDC
step=RawFileRsyncedISDC
rsyncpwfile=/home_nfs/isdc/fact_opr/rsync.pwd

set -C

logfile=$runlogpath"/RsyncRawToISDC-"$datetime".log"
date >> $logfile 2>&1


# check if only a certain date should be processed
#  (feature for manual speedup) 
if [ "$certaindate" != "" ]
then
   printprocesslog "INFO process only data of "$certaindate
   echo "INFO process only data of "$certaindate
   specialnight=`echo $certaindate | sed -e 's/\///g'`
fi


# stop transfer when less than 100 GB are left
function check_scratch()
{
   diskusage2=( `df -P /scratch | grep scratch ` )
   # check if more than X GB are left on /scratch
   if [ ${diskusage2[3]} -lt 100000 ] 
   then
      printprocesslog "DISK less than 100 GB left on /scratch ("${diskusage2[3]}")"
      echo "WARN less than 100 GB left on /scratch ("${diskusage2[3]}")" 
      finish
   fi
}

check_scratch

# get todo list
gettodo 

for (( s=0 ; s < $num ; s++ ))
do
   check_scratch
   night=${primaries[$s+$s]}
   runid=${primaries[$s+$s+1]}
   nightpath=`echo $night | cut -c 1-4`"/"`echo $night | cut -c 5-6`"/"`echo $night | cut -c 7-8`
   lppath="/data2/zipraw/"$nightpath
   nightrun=$night"_"`printf %03d $runid`

   # continue if only special night should be processed for manual speedup
   if [ "$specialnight" != "" ]
   then
      if [ $night -ne $specialnight ]
      then
         printprocesslog "INFO Treat only "$specialnight" -> continue for "$night
         echo "INFO Treat only "$specialnight" -> continue for "$night
         continue
      else
         echo "INFO Transfer "$nightrun
      fi
   fi

   lpfile=`ssh fact@161.72.93.138 "find $lppath -type f -regex '.*$nightrun[.]fits[.][fg]z'"`
   if [ "$lpfile" == "" ]
   then
      #echo $nightrun" not yet available on data."
      #printprocesslog "INFO "$nightrun" not yet available on data."
      if [ $night -le $checknight ] 
      then 
         printprocesslog "WARN "$nightrun" not yet available on data."
      else
         printprocesslog "INFO "$nightrun" not yet available on data."
      fi
      continue
   else
      filename=`basename $lpfile`
   fi

   setstatus "start" 

   rawfilelp="/data2/zipraw/./"$nightpath"/"$filename
   rawdirisdc="/scratch/from_lapalma/raw/"
   rawfileisdc=$rawdirisdc$nightpath"/"$filename
   ingestlinkpath="/scratch/ingest_links/raw/"$nightpath
   makedir $ingestlinkpath >> $logfile 2>&1
   ingestlink=$ingestlinkpath"/"$filename

   if ! ssh fact@161.72.93.138 "ls $rawfilelp >/dev/null 2>&1"
   then
      #echo $rawfilelp" not yet available on data."
      #printprocesslog "INFO "$rawfilelp" not yet available on data."
      if [ $night -le $checknight ] 
      then 
         printprocesslog "WARN "$rawfilelp" not yet available on data."
      else
         printprocesslog "INFO "$rawfilelp" not yet available on data."
      fi
      check="no"
      setstatus "stop"
      continue
   fi

   # for new rsync
   #   remove / from dir
   #   change name of dir (else rsync-server doesn't accept it
   #   test if trick with file/dir is working
   #echo "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc
   printprocesslog "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc
   #echo "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc >> $logfile 2>&1
   #rsync -rltDvR --partial --stats --password-file=$rsyncpwfile factdata@161.72.93.138::$rawfilelp $rawdirisdc >> $logfile 2>&1
   #echo "rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.138:$rawfilelp $rawdirisdc >> $logfile 2>&1"
   rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.138:$rawfilelp $rawdirisdc >> $logfile 2>&1
   check1=$?

   case $check1 in
      0)   printprocesslog "INFO rawfile "$rawfilelp" transferred successfully to ISDC."
           if ! [ -L $ingestlink ]
           then 
              printprocesslog "INFO create link for "$rawfileisdc
              ln -sv $rawfileisdc $ingestlink >> $logfile 2>&1
              check2=$?
              case $check2 in
                 0)   printprocesslog "INFO link for  "$rawfileisdc" created successfully."
                      ;;
                 *)   printprocesslog "WARN link for "$rawfileisdc" could not be created (check2="$check2")."
                      check=$check2
                      ;;
              esac
           else
              printprocesslog "INFO link "$ingestlink" already existing."
           fi
           ;;
      *)   printprocesslog "CONNECTION "$rawfilelp" could not be transferred to ISDC (check1="$check1")."
           check=$check1
           ;;
   esac
   
   query="SELECT fHasDrsFile from RunInfo WHERE fNight="$night" AND fRunID="$runid
   numdrs=`sendquery`
   if [ $numdrs -gt 0 ]
   then 
      drsfilelp="/data2/zipraw/./"$nightpath"/"$nightrun".drs.fits.gz"
      drsfileisdc=$rawdirisdc$nightpath"/"$nightrun".drs.fits.gz"
      drsingestlink=$ingestlinkpath"/"$nightrun".drs.fits.gz"

      if ! ssh fact@161.72.93.138 "ls $drsfilelp >/dev/null 2>&1"
      then
         #echo $drsfilelp" not yet available on data."
         printprocesslog "INFO "$drsfilelp" not yet available on data."
         check="no"
         setstatus "stop"
         continue
      fi


      printprocesslog "INFO rsync drsfile "$drsfilelp" to ISDC "$rawdirisdc
      echo "INFO rsync drsfile "$drsfilelp" to ISDC "$rawdirisdc >> $logfile 2>&1
      #rsync -rltDvR --partial --stats --password-file=$rsyncpwfile factdata@161.72.93.138::$rawfilelp $rawdirisdc >> $logfile 2>&1
      rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.138:$drsfilelp $rawdirisdc >> $logfile 2>&1
      check3=$?

      case $check3 in
         0)   printprocesslog "INFO drsfile "$drsfilelp" transferred successfully to ISDC."
              if ! [ -L $drsingestlink ]
              then 
                 printprocesslog "INFO create link for "$drsfileisdc
                 ln -sv $drsfileisdc $drsingestlink >> $logfile 2>&1
                 check4=$?
                 case $check4 in
                    0)   printprocesslog "INFO link for  "$drsfileisdc" created successfully."
                         ;;
                    *)   printprocesslog "WARN link for "$drsfileisdc" could not be created (check4="$check4")."
                         check=$check4
                         ;;
                 esac
              else
                 printprocesslog "INFO link "$drsingestlink" already existing."
              fi
              ;;
         *)   printprocesslog "CONNECTION "$drsfilelp" could not be transferred to ISDC (check3="$check3")."
              check=$check3
              ;;
      esac
   fi

   setstatus "stop" 
done

finish