#!/bin/bash # source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0" program=RsyncRawToISDC step=RawFileRsyncedISDC pwfile=/home_nfs/isdc/fact_opr/rsync.pwd set -C logfile=$runlogpath"/RsyncRawToISDC-"$datetime".log" date >> $logfile 2>&1 ## check if script is already running #lockfile=$lockpath/lock-$program.txt #checklock # stop transfer when less than 100 GB are left function check_scratch() { diskusage2=( `df -P /scratch | grep scratch ` ) # check if more than X GB are left on /scratch if [ ${diskusage2[3]} -lt 100000 ] then printprocesslog "DISK less than 100 GB left on /scratch ("${diskusage2[3]}")" #echo "WARN less than 100 GB left on /scratch ("${diskusage2[3]}")" echo "WARN less than 100 GB left on /scratch ("${diskusage2[3]}")" >> $logfile2 2>&1 finish fi } check_scratch # get todo list gettodo for (( s=0 ; s < $num ; s++ )) do check_scratch night=${primaries[$s+$s]} runid=${primaries[$s+$s+1]} nightpath=`echo $night | cut -c 1-4`"/"`echo $night | cut -c 5-6`"/"`echo $night | cut -c 7-8` lppath="/loc_data/zipraw/"$nightpath nightrun=$night"_"`printf %03d $runid` lpfile=`ssh fact@161.72.93.131 "find $lppath -type f -regex '.*$nightrun[.]fits[.][fg]z'"` if [ "$lpfile" == "" ] then #echo $nightrun" not yet available on data." printprocesslog "INFO "$nightrun" not yet available on data." continue else filename=`basename $lpfile` fi setstatus "start" rawfilelp="/loc_data/zipraw/./"$nightpath"/"$filename rawdirisdc="/scratch/from_lapalma/raw/" rawfileisdc=$rawdirisdc$nightpath"/"$filename ingestlinkpath="/scratch/ingest_links/raw/"$nightpath makedir $ingestlinkpath >> $logfile 2>&1 ingestlink=$ingestlinkpath"/"$filename if ! ssh fact@161.72.93.131 "ls $rawfilelp >/dev/null 2>&1" then #echo $rawfilelp" not yet available on data." printprocesslog "INFO "$rawfilelp" not yet available on data." check="no" setstatus "stop" continue fi # for new rsync # remove / from dir # change name of dir (else rsync-server doesn't accept it # test if trick with file/dir is working #echo "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc printprocesslog "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc #echo "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc >> $logfile 2>&1 #rsync -rltDvR --partial --stats --password-file=$pwfile factdata@161.72.93.131::$rawfilelp $rawdirisdc >> $logfile 2>&1 #echo "rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.131:$rawfilelp $rawdirisdc >> $logfile 2>&1" rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.131:$rawfilelp $rawdirisdc >> $logfile 2>&1 check1=$? case $check1 in 0) printprocesslog "INFO rawfile "$rawfilelp" transferred successfully to ISDC." if ! [ -L $ingestlink ] then printprocesslog "INFO create link for "$rawfileisdc ln -sv $rawfileisdc $ingestlink >> $logfile 2>&1 check2=$? case $check2 in 0) printprocesslog "INFO link for "$rawfileisdc" created successfully." ;; *) printprocesslog "WARN link for "$rawfileisdc" could not be created (check2="$check2")." check=$check2 ;; esac else printprocesslog "INFO link "$ingestlink" already existing." fi ;; *) printprocesslog "CONNECTION "$rawfilelp" could not be transferred to ISDC (check1="$check1")." check=$check1 ;; esac query="SELECT fHasDrsFile from RunInfo WHERE fNight="$night" AND fRunID="$runid numdrs=`sendquery` if [ $numdrs -gt 0 ] then drsfilelp="/loc_data/zipraw/./"$nightpath"/"$nightrun".drs.fits.gz" drsfileisdc=$rawdirisdc$nightpath"/"$nightrun".drs.fits.gz" drsingestlink=$ingestlinkpath"/"$nightrun".drs.fits.gz" if ! ssh fact@161.72.93.131 "ls $drsfilelp >/dev/null 2>&1" then #echo $drsfilelp" not yet available on data." printprocesslog "INFO "$drsfilelp" not yet available on data." check="no" setstatus "stop" continue fi printprocesslog "INFO rsync drsfile "$drsfilelp" to ISDC "$rawdirisdc echo "INFO rsync drsfile "$drsfilelp" to ISDC "$rawdirisdc >> $logfile 2>&1 #rsync -rltDvR --partial --stats --password-file=$pwfile factdata@161.72.93.131::$rawfilelp $rawdirisdc >> $logfile 2>&1 rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.131:$drsfilelp $rawdirisdc >> $logfile 2>&1 check3=$? case $check3 in 0) printprocesslog "INFO drsfile "$drsfilelp" transferred successfully to ISDC." if ! [ -L $drsingestlink ] then printprocesslog "INFO create link for "$drsfileisdc ln -sv $drsfileisdc $drsingestlink >> $logfile 2>&1 check4=$? case $check4 in 0) printprocesslog "INFO link for "$drsfileisdc" created successfully." ;; *) printprocesslog "WARN link for "$drsfileisdc" could not be created (check4="$check4")." check=$check4 ;; esac else printprocesslog "INFO link "$drsingestlink" already existing." fi ;; *) printprocesslog "CONNECTION "$drsfilelp" could not be transferred to ISDC (check3="$check3")." check=$check3 ;; esac fi setstatus "stop" done finish