Ignore:
Timestamp:
05/14/17 20:59:09 (8 years ago)
Author:
Daniela Dorner
Message:
updated to run on newdata and create a second copy
File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/DataCheck/Transfer/RsyncAuxLP.sh

    r18671 r18854  
    1919
    2020# check if paths are available
    21 if ! ls /data/aux >/dev/null 2>&1
    22 then
    23    printprocesslog "ERROR /data/aux is not available."
    24    finish
    25 fi
    26 if ! ls /loc_data/aux >/dev/null 2>&1
    27 then
    28    printprocesslog "ERROR /loc_data/aux is not available."
    29    finish
     21paths="/newdaq/aux /data1/aux /data2/aux /scratch/aux"
     22for path in ${paths[@]}
     23do
     24   if ! ls $path >/dev/null 2>&1
     25   then
     26      printprocesslog "ERROR "$path" is not available."
     27      finish
     28   fi
     29done
     30
     31# first get some settings
     32# get current hour
     33hour=`date +%k`
     34# define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync
     35if [ $hour -le 6 ] || [ $hour -ge 19 ]
     36then
     37   # limit bw for rsync to 20 MB/s during night
     38   bwlimit="--bwlimit=20000"
     39   bwlimit="--bwlimit=5000"
     40   printprocesslog "INFO rsync files with "$bwlimit
     41   echo "rsync files with "$bwlimit >> $logfile 2>&1
     42   searchwindow="-cmin -840"
     43   printprocesslog "INFO treat the last 14 hours."
     44   echo "INFO treat the last 14 hours."  >> $logfile 2>&1
     45else
     46   # no bw limit during day
     47   bwlimit=""
     48   printprocesslog "INFO rsync files without bwlimit"
     49   echo "rsync files without bwlimit" >> $logfile 2>&1
     50   searchwindow="-ctime -6"
     51   printprocesslog "INFO treat the last 14 hours."
     52   echo "INFO treat the last 14 hours."  >> $logfile 2>&1
    3053fi
    3154
    32 # do the rsync for the last 6 days
    33 dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 -type d | sort | tail -6"` )
     55# find directories to be rsynced
     56dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 $searchwindow -type d | sort "` )
    3457
    3558# do rsync for auxfiles in these directories
     
    3760do
    3861   echo "" >> $logfile 2>&1
    39    # directory on daq
    40    if ! [ -d $dir ]
     62   printprocesslog "INFO processing files in "$dir
     63   echo `date`": processing files in "$dir >> $logfile 2>&1
     64   #echo `date`": processing files in "$dir
     65   
     66   # first take care of SSD /scratch
     67   scratch=`echo $dir | sed -e 's/loc_data/scratch/'`
     68   if ! [ -d $scratch ]
    4169   then
    42       mkdir -pv $dir >> $logfile 2>&1
     70      mkdir -pv $scratch >> $logfile 2>&1
    4371   fi
    44    # directory on data
    45    dirdata=`echo $dir | sed -e 's/loc_//'`
    46    if ! [ -d $dirdata ]
     72   # rsync from newdaq to newdata
     73   if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit newdaq:$dir/ $scratch >> $logfile 2>&1
     74   then
     75      printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to newdata"
     76      echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to newdata" >> $logfile 2>&1
     77   fi
     78   
     79   # now make copies on newdata
     80   data1=`echo $dir | sed -e 's/loc_data/data1/'`
     81   if ! [ -d $data1 ]
    4782   then
    48       mkdir -pv $dirdata >> $logfile 2>&1
     83      mkdir -pv $data1 >> $logfile 2>&1
    4984   fi
    50    printprocesslog "INFO processing files in "$dir >> $logfile 2>&1
    51    echo `date`": processing files in "$dir >> $logfile 2>&1
    52    
    53    # get current hour
    54    hour=`date +%k`
    55    # define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync
    56    if [ $hour -le 6 ] || [ $hour -ge 19 ]
     85   data2=`echo $dir | sed -e 's/loc_data/data2/'`
     86   if ! [ -d $data2 ]
     87   then
     88      mkdir -pv $data2 >> $logfile 2>&1
     89   fi
     90   # distribute to several disk on newdata
     91   if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit $scratch/ $data1 >> $logfile 2>&1
    5792   then
    58       # limit bw for rsync to 20 MB/s during night
    59       bwlimit="--bwlimit=20000"
    60       bwlimit="--bwlimit=5000"
    61       printprocesslog "INFO rsync files with "$bwlimit >> $logfile 2>&1
    62       echo "rsync files with "$bwlimit >> $logfile 2>&1
    63    else
    64       # no bw limit during day
    65       bwlimit=""
    66       printprocesslog "INFO rsync files without bwlimit" >> $logfile 2>&1
    67       echo "rsync files without bwlimit" >> $logfile 2>&1
     93      printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data1"
     94      echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data1" >> $logfile 2>&1
    6895   fi
    69 
    70    # rsync from newdaq to daq
    71    rsyncserverdir=`echo $dir | sed -e 's/^\//172.16.100.100::/' -e 's/loc_data/newdaq/'`
    72    # old
    73    if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit newdaq:$dir/ $dir >> $logfile 2>&1
    74    # new (temporary until pb on daq is solved)
    75    #if ! /usr/bin/rsync -avxHP -T $rsynctempdir $bwlimit $rsyncserverdir/ $dir >> $logfile 2>&1
     96   if ! /usr/bin/rsync -avu -T $rsynctempdir $bwlimit $scratch/ $data2 >> $logfile 2>&1
    7697   then
    77       printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to daq"
    78       echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to daq" >> $logfile 2>&1
    79    fi
    80 
    81    # rsynctemp-dir -> use one on data!!! i.e. has to be /data/rsync_tmp
    82    # rsync from daq to data
    83    rsynctempdirdata=/data/rsync_tmp
    84    if ! /usr/bin/rsync -avxHP -T $rsynctempdirdata $bwlimit $dir/ $dirdata >> $logfile 2>&1
    85    then
    86       printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from daq to data"
    87       echo `date`": problem rsyncing auxiliary files in "$dir" from daq to data" >> $logfile 2>&1
     98      printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data2"
     99      echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data2" >> $logfile 2>&1
    88100   fi
    89101done
    90102
     103finish
Note: See TracChangeset for help on using the changeset viewer.