Changeset 18854 for trunk/DataCheck/Transfer/RsyncAuxLP.sh

Timestamp:

05/14/17 20:59:09 (8 years ago)

Author:

Daniela Dorner

Message:

updated to run on newdata and create a second copy

File:

• trunk/DataCheck/Transfer/RsyncAuxLP.sh (modified) (2 diffs)

trunk/DataCheck/Transfer/RsyncAuxLP.sh

@@ -19 +19 @@
 
 # check if paths are available
-if ! ls /data/aux >/dev/null 2>&1
-then 
-   printprocesslog "ERROR /data/aux is not available."
-   finish
-fi
-if ! ls /loc_data/aux >/dev/null 2>&1
-then 
-   printprocesslog "ERROR /loc_data/aux is not available."
-   finish
+paths="/newdaq/aux /data1/aux /data2/aux /scratch/aux"
+for path in ${paths[@]}
+do 
+   if ! ls $path >/dev/null 2>&1
+   then 
+      printprocesslog "ERROR "$path" is not available."
+      finish
+   fi
+done
+
+# first get some settings
+# get current hour
+hour=`date +%k`
+# define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync
+if [ $hour -le 6 ] || [ $hour -ge 19 ]
+then
+   # limit bw for rsync to 20 MB/s during night
+   bwlimit="--bwlimit=20000"
+   bwlimit="--bwlimit=5000"
+   printprocesslog "INFO rsync files with "$bwlimit 
+   echo "rsync files with "$bwlimit >> $logfile 2>&1
+   searchwindow="-cmin -840"
+   printprocesslog "INFO treat the last 14 hours."
+   echo "INFO treat the last 14 hours."  >> $logfile 2>&1
+else 
+   # no bw limit during day
+   bwlimit=""
+   printprocesslog "INFO rsync files without bwlimit"
+   echo "rsync files without bwlimit" >> $logfile 2>&1
+   searchwindow="-ctime -6"
+   printprocesslog "INFO treat the last 14 hours."
+   echo "INFO treat the last 14 hours."  >> $logfile 2>&1
 fi
 
-# do the rsync for the last 6 days
-dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 -type d | sort | tail -6"` )
+# find directories to be rsynced
+dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 $searchwindow -type d | sort "` )
 
 # do rsync for auxfiles in these directories
@@ -37 +60 @@
 do 
    echo "" >> $logfile 2>&1
-   # directory on daq
-   if ! [ -d $dir ]
+   printprocesslog "INFO processing files in "$dir 
+   echo `date`": processing files in "$dir >> $logfile 2>&1
+   #echo `date`": processing files in "$dir
+   
+   # first take care of SSD /scratch
+   scratch=`echo $dir | sed -e 's/loc_data/scratch/'`
+   if ! [ -d $scratch ]
    then 
-      mkdir -pv $dir >> $logfile 2>&1
+      mkdir -pv $scratch >> $logfile 2>&1
    fi
-   # directory on data
-   dirdata=`echo $dir | sed -e 's/loc_//'`
-   if ! [ -d $dirdata ]
+   # rsync from newdaq to newdata
+   if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit newdaq:$dir/ $scratch >> $logfile 2>&1
+   then
+      printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to newdata"
+      echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to newdata" >> $logfile 2>&1
+   fi
+   
+   # now make copies on newdata
+   data1=`echo $dir | sed -e 's/loc_data/data1/'`
+   if ! [ -d $data1 ]
    then 
-      mkdir -pv $dirdata >> $logfile 2>&1
+      mkdir -pv $data1 >> $logfile 2>&1
    fi
-   printprocesslog "INFO processing files in "$dir >> $logfile 2>&1
-   echo `date`": processing files in "$dir >> $logfile 2>&1
-   
-   # get current hour
-   hour=`date +%k`
-   # define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync
-   if [ $hour -le 6 ] || [ $hour -ge 19 ]
+   data2=`echo $dir | sed -e 's/loc_data/data2/'`
+   if ! [ -d $data2 ]
+   then 
+      mkdir -pv $data2 >> $logfile 2>&1
+   fi
+   # distribute to several disk on newdata
+   if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit $scratch/ $data1 >> $logfile 2>&1
    then
-      # limit bw for rsync to 20 MB/s during night
-      bwlimit="--bwlimit=20000"
-      bwlimit="--bwlimit=5000"
-      printprocesslog "INFO rsync files with "$bwlimit >> $logfile 2>&1
-      echo "rsync files with "$bwlimit >> $logfile 2>&1
-   else 
-      # no bw limit during day
-      bwlimit=""
-      printprocesslog "INFO rsync files without bwlimit" >> $logfile 2>&1
-      echo "rsync files without bwlimit" >> $logfile 2>&1
+      printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data1"
+      echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data1" >> $logfile 2>&1
    fi
-
-   # rsync from newdaq to daq
-   rsyncserverdir=`echo $dir | sed -e 's/^\//172.16.100.100::/' -e 's/loc_data/newdaq/'` 
-   # old
-   if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit newdaq:$dir/ $dir >> $logfile 2>&1
-   # new (temporary until pb on daq is solved)
-   #if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit $rsyncserverdir/ $dir >> $logfile 2>&1
+   if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit $scratch/ $data2 >> $logfile 2>&1
    then
-      printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to daq"
-      echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to daq" >> $logfile 2>&1
-   fi
-
-   # rsynctemp-dir -> use one on data!!! i.e. has to be /data/rsync_tmp
-   # rsync from daq to data
-   rsynctempdirdata=/data/rsync_tmp
-   if ! /usr/bin/rsync -avxHP -T $rsynctempdirdata $bwlimit $dir/ $dirdata >> $logfile 2>&1
-   then
-      printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from daq to data"
-      echo `date`": problem rsyncing auxiliary files in "$dir" from daq to data" >> $logfile 2>&1
+      printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data2"
+      echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data2" >> $logfile 2>&1
    fi
 done
 
+finish