Changeset 18854 for trunk/DataCheck/Transfer
- Timestamp:
- 05/14/17 20:59:09 (8 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/DataCheck/Transfer/RsyncAuxLP.sh
r18671 r18854 19 19 20 20 # check if paths are available 21 if ! ls /data/aux >/dev/null 2>&1 22 then 23 printprocesslog "ERROR /data/aux is not available." 24 finish 25 fi 26 if ! ls /loc_data/aux >/dev/null 2>&1 27 then 28 printprocesslog "ERROR /loc_data/aux is not available." 29 finish 21 paths="/newdaq/aux /data1/aux /data2/aux /scratch/aux" 22 for path in ${paths[@]} 23 do 24 if ! ls $path >/dev/null 2>&1 25 then 26 printprocesslog "ERROR "$path" is not available." 27 finish 28 fi 29 done 30 31 # first get some settings 32 # get current hour 33 hour=`date +%k` 34 # define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync 35 if [ $hour -le 6 ] || [ $hour -ge 19 ] 36 then 37 # limit bw for rsync to 20 MB/s during night 38 bwlimit="--bwlimit=20000" 39 bwlimit="--bwlimit=5000" 40 printprocesslog "INFO rsync files with "$bwlimit 41 echo "rsync files with "$bwlimit >> $logfile 2>&1 42 searchwindow="-cmin -840" 43 printprocesslog "INFO treat the last 14 hours." 44 echo "INFO treat the last 14 hours." >> $logfile 2>&1 45 else 46 # no bw limit during day 47 bwlimit="" 48 printprocesslog "INFO rsync files without bwlimit" 49 echo "rsync files without bwlimit" >> $logfile 2>&1 50 searchwindow="-ctime -6" 51 printprocesslog "INFO treat the last 14 hours." 52 echo "INFO treat the last 14 hours." >> $logfile 2>&1 30 53 fi 31 54 32 # do the rsync for the last 6 days33 dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 -type d | sort | tail -6"` )55 # find directories to be rsynced 56 dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 $searchwindow -type d | sort "` ) 34 57 35 58 # do rsync for auxfiles in these directories … … 37 60 do 38 61 echo "" >> $logfile 2>&1 39 # directory on daq 40 if ! [ -d $dir ] 62 printprocesslog "INFO processing files in "$dir 63 echo `date`": processing files in "$dir >> $logfile 2>&1 64 #echo `date`": processing files in "$dir 65 66 # first take care of SSD /scratch 67 scratch=`echo $dir | sed -e 's/loc_data/scratch/'` 68 if ! [ -d $scratch ] 41 69 then 42 mkdir -pv $ dir>> $logfile 2>&170 mkdir -pv $scratch >> $logfile 2>&1 43 71 fi 44 # directory on data 45 dirdata=`echo $dir | sed -e 's/loc_//'` 46 if ! [ -d $dirdata ] 72 # rsync from newdaq to newdata 73 if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit newdaq:$dir/ $scratch >> $logfile 2>&1 74 then 75 printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to newdata" 76 echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to newdata" >> $logfile 2>&1 77 fi 78 79 # now make copies on newdata 80 data1=`echo $dir | sed -e 's/loc_data/data1/'` 81 if ! [ -d $data1 ] 47 82 then 48 mkdir -pv $d irdata>> $logfile 2>&183 mkdir -pv $data1 >> $logfile 2>&1 49 84 fi 50 printprocesslog "INFO processing files in "$dir >> $logfile 2>&151 echo `date`": processing files in "$dir >> $logfile 2>&152 53 # get current hour54 hour=`date +%k`55 # d efine bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync56 if [ $hour -le 6 ] || [ $hour -ge 19 ]85 data2=`echo $dir | sed -e 's/loc_data/data2/'` 86 if ! [ -d $data2 ] 87 then 88 mkdir -pv $data2 >> $logfile 2>&1 89 fi 90 # distribute to several disk on newdata 91 if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit $scratch/ $data1 >> $logfile 2>&1 57 92 then 58 # limit bw for rsync to 20 MB/s during night 59 bwlimit="--bwlimit=20000" 60 bwlimit="--bwlimit=5000" 61 printprocesslog "INFO rsync files with "$bwlimit >> $logfile 2>&1 62 echo "rsync files with "$bwlimit >> $logfile 2>&1 63 else 64 # no bw limit during day 65 bwlimit="" 66 printprocesslog "INFO rsync files without bwlimit" >> $logfile 2>&1 67 echo "rsync files without bwlimit" >> $logfile 2>&1 93 printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data1" 94 echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data1" >> $logfile 2>&1 68 95 fi 69 70 # rsync from newdaq to daq 71 rsyncserverdir=`echo $dir | sed -e 's/^\//172.16.100.100::/' -e 's/loc_data/newdaq/'` 72 # old 73 if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit newdaq:$dir/ $dir >> $logfile 2>&1 74 # new (temporary until pb on daq is solved) 75 #if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit $rsyncserverdir/ $dir >> $logfile 2>&1 96 if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit $scratch/ $data2 >> $logfile 2>&1 76 97 then 77 printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to daq" 78 echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to daq" >> $logfile 2>&1 79 fi 80 81 # rsynctemp-dir -> use one on data!!! i.e. has to be /data/rsync_tmp 82 # rsync from daq to data 83 rsynctempdirdata=/data/rsync_tmp 84 if ! /usr/bin/rsync -avxHP -T $rsynctempdirdata $bwlimit $dir/ $dirdata >> $logfile 2>&1 85 then 86 printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from daq to data" 87 echo `date`": problem rsyncing auxiliary files in "$dir" from daq to data" >> $logfile 2>&1 98 printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data2" 99 echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data2" >> $logfile 2>&1 88 100 fi 89 101 done 90 102 103 finish
Note:
See TracChangeset
for help on using the changeset viewer.