Changeset 14841 for trunk/DataCheck/Transfer

Timestamp:

02/04/13 06:03:36 (12 years ago)

Author:

Daniela Dorner

Message:

adapted to new structure in LP (newdaq, new transfer scheme)

File:

• trunk/DataCheck/Transfer/RsyncAuxLP.sh (modified) (4 diffs)

trunk/DataCheck/Transfer/RsyncAuxLP.sh

@@ -1 +1 @@
 #!/bin/bash
 
-# this script has been written to run on La Palma on the machine data
+# this script has been written to run on La Palma on the machine daq
 #   i.e. paths are only working on this machine
 
@@ -11 +11 @@
 
 # check if /daq is mounted on data
-if ! mount | grep daq >> $logfile 2>&1
+if ! mount | grep data >> $logfile 2>&1
 then 
-   printprocesslog "ERROR /daq is not mounted on data => please mount it"
-   echo `date`": /daq is not mounted on data => please mount it"
+   printprocesslog "ERROR /data is not mounted on daq => please mount it"
+   echo `date`": /data is not mounted on daq => please mount it"
    finish
 fi
 
 # check if paths are available
-if ! ls /daq/aux >/dev/null 2>&1
+if ! ls /data/aux >/dev/null 2>&1
 then 
-   printprocesslog "ERROR /daq/aux is not available."
+   printprocesslog "ERROR /data/aux is not available."
    finish
 fi
@@ -30 +30 @@
 fi
 
+# do the rsync for the last 6 days
+dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 -type d | sort | tail -6"` )
 
-# get last 3, 6 or 9 nights
-dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \
-        `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \
-        `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \
-        )
-
-# do rsync for rawfiles of these dates
-for date in ${dates[@]}
+# do rsync for auxfiles in these directories
+for dir in ${dirs[@]}
 do 
    echo "" >> $logfile 2>&1
-   # auxiliary data directories on data 
-   auxdirdaq=/daq/aux/$date/ # /loc_data from daq via nfs on data
-   if ! [ -d $auxdirdaq ]
+   # directory on daq
+   if ! [ -d $dir ]
    then 
-      auxdirdata=/loc_data/aux/$date/
-      if ! [ -d $auxdirdata ]
-      then 
-         printprocesslog "INFO "$auxdirdaq" and "$auxdirdata" not found. " >> $logfile 2>&1
-      else
-         printprocesslog "WARN "$auxdirdaq" not found, but "$auxdirdata". Data were probably taken on data." >> $logfile 2>&1
-      fi
-      echo `date`": "$auxdirdaq" not found. " >> $logfile 2>&1
-      continue
+      mkdir -pv $dir >> $logfile 2>&1
    fi
-   auxdirdata=/loc_data/aux/$date 
-   if ! [ -d $auxdirdata ]
+   # directory on data
+   dirdata=`echo $dir | sed -e 's/loc_//'`
+   if ! [ -d $dirdata ]
    then 
-      mkdir -pv $auxdirdata >> $logfile 2>&1
+      mkdir -pv $dirdata >> $logfile 2>&1
    fi
-   printprocesslog "INFO processing files in "$auxdirdaq >> $logfile 2>&1
-   echo `date`": processing files in "$auxdirdaq >> $logfile 2>&1
+   printprocesslog "INFO processing files in "$dir >> $logfile 2>&1
+   echo `date`": processing files in "$dir >> $logfile 2>&1
    
    # get current hour
@@ -70 +58 @@
       # limit bw for rsync to 20 MB/s during night
       bwlimit="--bwlimit=20000"
-      printprocesslog "INFO rsync data with "$bwlimit >> $logfile 2>&1
-      echo "rsync data with "$bwlimit >> $logfile 2>&1
+      printprocesslog "INFO rsync files with "$bwlimit >> $logfile 2>&1
+      echo "rsync files with "$bwlimit >> $logfile 2>&1
    else 
       # no bw limit during day
-      printprocesslog "INFO rsync data without bwlimit" >> $logfile 2>&1
-      echo "rsync data without bwlimit" >> $logfile 2>&1
+      printprocesslog "INFO rsync files without bwlimit" >> $logfile 2>&1
+      echo "rsync files without bwlimit" >> $logfile 2>&1
+   fi
+
+   # rsync from newdaq to daq
+   if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit newdaq:$dir/ $dir >> $logfile 2>&1
+   then
+      printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from newdaq to daq"
+      echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to daq"
    fi
    
    #rsync from daq to data
-   #if ! /usr/bin/rsync -avxHPu $bwlimit $auxdirdaq $auxdirdata >> $logfile 2>&1 # in case of adding checksum, only update files with rsync
-   if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit $auxdirdaq $auxdirdata >> $logfile 2>&1
+   if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit $dir/ $dirdata >> $logfile 2>&1
    then
-      printprocesslog "WARN problem rsyncing auxiliary data for "$date" from daq to data"
-      echo `date`": problem rsyncing auxiliary data for "$date" from daq to data"
+      printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from daq to data"
+      echo `date`": problem rsyncing auxiliary files in "$dir" from daq to data"
    fi
 done