#!/bin/bash # source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0" program=RsyncRawToWue step=RawFileAvailWue set -C # check if only a certain date should be processed # (feature for manual speedup) if [ "$certaindate" != "" ] then checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'` if [ "$checkstring" = "" ] then echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)" finish fi printprocesslog "INFO process only data of "$certaindate echo "INFO process only data of "$certaindate specialnight=`echo $certaindate | sed -e 's/\///g'` fi # checking that not more than $numrsyncwuelimit # instances of the script are running numrsyncwue=`/usr/sbin/lsof $0 | grep -o -c $0` if [ $numrsyncwue -gt $numrsyncwuelimit ] then printprocesslog "INFO already "$numrsyncwue" BackupRawToWue.sh running -> exit. " finish fi logfile=$runlogpath"/RsyncRawToWue-"$datetime".log" date >> $logfile # get todo list gettodo for (( s=0 ; s < $num ; s++ )) do night=${primaries[$s+$s]} runid=${primaries[$s+$s+1]} nightpath=`echo $night | cut -c 1-4`"/"`echo $night | cut -c 5-6`"/"`echo $night | cut -c 7-8` nightrun=$night"_"`printf %03d $runid` if [ "$specialnight" != "" ] then if [ $night -ne $specialnight ] then printprocesslog "INFO Treat only "$specialnight" -> continue for "$night echo "INFO Treat only "$specialnight" -> continue for "$night continue fi fi rawdirisdc="/scratch/from_lapalma/raw" isdcfile=`find $rawdirisdc -type f -regex ".*$nightrun[.]fits[.][fg]z"` if [ "$isdcfile" == "" ] then echo "ERROR Could not find raw file for "$nightrun" on isdc-dl00 - Something is really wrong." printprocesslog "ERROR Could not find raw file for "$nightrun" on isdc-dl00 - Something is really wrong." continue else filename=`basename $isdcfile` fi rawfileisdc=$rawdirisdc"/./"$nightpath"/"$filename rawdirwue="/fact/raw/" setstatus "start" if [ $numchanged -eq 0 ] then printprocesslog "INFO "$numchanged" rows were changed in DB => "$night"_"$runid" is already processed => continue." continue fi printprocesslog "INFO rsync rawfile "$rawfileisdc" to Wue "$rawdirwue #rsync -rltDvR --partial --stats --password-file=/home_nfs/isdc/fact_opr/rsync.pwd factdata@161.72.93.131::$rawfilelp $rawdirisdc >> $logfile 2>&1 #echo "rsync -avxR --no-p --stats -T $rsynctempdir $rawfileisdc operator@coma.astro.uni-wuerzburg.de:$rawdirwue >> $logfile 2>&1" rsync -avxR --no-p --stats -T $rsynctempdir $rawfileisdc operator@coma.astro.uni-wuerzburg.de:$rawdirwue >> $logfile 2>&1 check1=$? case $check1 in 0) printprocesslog "INFO rawfile "$rawfileisdc" transferred successfully to Wue." ;; *) printprocesslog "CONNECTION "$rawfileisdc" could not be transferred to Wue." check=$check1 ;; esac query="SELECT fHasDrsFile from RunInfo WHERE fNight="$night" AND fRunID="$runid numdrs=`sendquery` if [ $numdrs -gt 0 ] then drsfileisdc=$rawdirisdc"/./"$nightpath"/"$nightrun".drs.fits.gz" printprocesslog "INFO rsync rawfile "$drsfileisdc" to Wue "$rawdirwue rsync -avxR --no-p --stats $drsfileisdc operator@coma.astro.uni-wuerzburg.de:$rawdirwue >> $logfile 2>&1 check1=$? case $check1 in 0) printprocesslog "INFO rawfile "$drsfileisdc" transferred successfully to Wue." ;; *) printprocesslog "CONNECTION "$drsfileisdc" could not be transferred to Wue." check=$check1 ;; esac fi setstatus "stop" done finish