Changeset 13054 for trunk/DataCheck/Transfer

Timestamp:

03/09/12 22:18:35 (13 years ago)

Author:

Daniela Dorner

Message:

reworked version of scripts doing backup to Wuerzburg

Location:

trunk/DataCheck/Transfer

Files:

• BackupAuxToWue.sh (modified) (1 diff)
• BackupRawToWue.sh (modified) (1 diff)

trunk/DataCheck/Transfer/BackupAuxToWue.sh

@@ -1 +1 @@
 #!/bin/bash
 
-isdcauxdata=/data00/fact-construction/aux/
-wueauxdata=/fact/aux
+# this script has been written to run on La Palma on the machine data
+#   i.e. paths are only working on this machine
 
-today=`date +%F-%H-%M-%S`
-logfile=~/DataCheck/log/BackupAuxToWue$today.log
+source `dirname $0`/../Sourcefile.sh
+printprocesslog "INFO starting "$0
 
-echo "" >> $logfile 2>&1
-echo "" >> $logfile 2>&1
-echo "" >> $logfile 2>&1
-echo `date`": start rsync of auxiliary data to Wuerzburg" >> $logfile 2>&1
+logfile=$runlogpath"/RsyncAuxToWue-"$datetime".log"
+date >> $logfile
 
-#rsync from ISDC to Wue DC
-if ! /usr/bin/rsync -avxH --no-p $isdcauxdata operator@coma.astro.uni-wuerzburg.de:$wueauxdata >> $logfile 2>&1
-then
-   echo `date`": problem rsyncing auxiliary data from ISDC ("$HOST") to Wuerzburg"
-fi
+# get last 3 nights
+dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
 
+# do rsync for rawfiles of these dates
+for date in ${dates[@]}
+do 
+   echo "" >> $logfile 2>&1
+   # auxiliary data directories
+   auxdirisdc=/scratch/from_lapalma/aux/./$date/ 
+   auxdirwue=/fact/aux
+   
+   if ! [ -d $auxdirisdc ]
+   then 
+      printprocesslog "INFO "$auxdirisdc" not available." >> $logfile 2>&1
+      continue
+   fi
+   printprocesslog "INFO processing files in "$auxdirisdc >> $logfile 2>&1
+   echo `date`": processing files in "$auxdirisdc >> $logfile 2>&1
+   
+   #rsync from ISDC to Wue
+   if ! rsync -rltDvR --stats $auxdirisdc operator@coma.astro.uni-wuerzburg.de:$auxdirwue >> $logfile 2>&1
+   then
+      printprocesslog "WARN problem rsyncing auxiliary data for "$date" from ISDC to Wue"
+      echo `date`": problem rsyncing auxiliary data for "$date" from ISDC to Wue" >> $logfile 2>&1
+      echo `date`": problem rsyncing auxiliary data for "$date" from ISDC to Wue"
+   fi
+done
 

trunk/DataCheck/Transfer/BackupRawToWue.sh

@@ -1 @@
-#!/bin/bash
+#!/bin/sh
+#
 
-isdcrawdata=/data00/fact-construction/raw/
-wuerawdata=/fact/raw
+source `dirname $0`/../Sourcefile.sh
+printprocesslog "INFO starting $0"
+program=RsyncRawToWue
+step=RawFileAvailWue
 
-today=`date +%F-%H-%M-%S`
-logfile=~/DataCheck/log/BackupRawToWue$today.log
+set -C
 
-echo "" >> $logfile 2>&1
-echo "" >> $logfile 2>&1
-echo "" >> $logfile 2>&1
-echo `date`": start rsync of raw data to Wuerzburg" >> $logfile 2>&1
+logfile=$runlogpath"/RsyncRawToWue-"$datetime".log"
+date >> $logfile
 
-#rsync from ISDC to Wue DC
-if ! /usr/bin/rsync -avx --no-p --progress $isdcrawdata operator@coma.astro.uni-wuerzburg.de:$wuerawdata >> $logfile 2>&1
-then
-   echo `date`": problem rsyncing rawdata from ISDC ("$HOST") to Wuerzburg"
-fi
+# check if script is already running
+lockfile=$lockpath/lock-$program.txt
+checklock 
+
+# get todo list
+gettodo 
 
 
+for (( s=0 ; s < $num ; s++ ))
+do
+   night=${primaries[$s+$s]}
+   runid=${primaries[$s+$s+1]}
+   nightpath=`echo $night | cut -c 1-4`"/"`echo $night | cut -c 5-6`"/"`echo $night | cut -c 7-8`
+   
+   setstatus "start" 
+
+   rawfileisdc="/scratch/from_lapalma/raw/./"$nightpath"/"$night"_"`printf %03d $runid`".fits.gz"
+   rawdirwue="/fact/raw/"
+
+   printprocesslog "INFO rsync rawfile "$rawfileisdc" to Wue "$rawdirwue
+   #rsync -rltDvR --partial --stats --password-file=/home_nfs/isdc/fact_opr/rsync.pwd factdata@161.72.93.131::$rawfilelp $rawdirisdc >> $logfile 2>&1
+   rsync -avxR --no-p --stats $rawfileisdc operator@coma.astro.uni-wuerzburg.de:$rawdirwue >> $logfile 2>&1
+   check1=$?
+
+   case $check1 in
+      0)   printprocesslog "INFO rawfile "$rawfileisdc" transferred successfully to Wue."
+           ;;
+      *)   printprocesslog "WARN "$rawfileisdc" could not be transferred to Wue."
+           check=$check1
+           ;;
+   esac
+   
+
+   setstatus "stop" 
+done
+
+finish 
+