source: trunk/DataCheck/Transfer/RsyncAuxLP.sh@ 19562

Last change on this file since 19562 was 18871, checked in by Daniela Dorner, 8 years ago
adapted to use proper rsync-temp-dir for each transfer
  • Property svn:executable set to *
File size: 3.6 KB
Line 
1#!/bin/bash
2
3# this script has been written to run on La Palma on the machine daq
4# i.e. paths are only working on this machine
5
6source `dirname $0`/../Sourcefile.sh
7printprocesslog "INFO starting "$0
8
9logfile=$runlogpath"/RsyncAuxLP-"$datetime".log"
10date >> $logfile
11
12# check if /daq is mounted on data
13if ! mount | grep data >> $logfile 2>&1
14then
15 printprocesslog "ERROR /data is not mounted on daq => please mount it"
16 echo `date`": /data is not mounted on daq => please mount it"
17 finish
18fi
19
20rsynctempdir1=/scratch/rsync_tmp
21if ! [ -d $rsynctempdir1 ]
22then
23 mkdir $rsynctempdir1
24fi
25rsynctempdir2=/scratch/rsync_tmp
26if ! [ -d $rsynctempdir2 ]
27then
28 mkdir $rsynctempdir2
29fi
30rsynctempdir3=/scratch/rsync_tmp
31if ! [ -d $rsynctempdir3 ]
32then
33 mkdir $rsynctempdir3
34fi
35
36# check if paths are available
37paths="/newdaq/aux /data1/aux /data2/aux /scratch/aux"
38for path in ${paths[@]}
39do
40 if ! ls $path >/dev/null 2>&1
41 then
42 printprocesslog "ERROR "$path" is not available."
43 finish
44 fi
45done
46
47# first get some settings
48# get current hour
49hour=`date +%k`
50# define bwlimit for rsync depending on the time: from 19-7h reduced bwlimit for rsync
51if [ $hour -le 6 ] || [ $hour -ge 19 ]
52then
53 # limit bw for rsync to 20 MB/s during night
54 bwlimit="--bwlimit=20000"
55 bwlimit="--bwlimit=5000"
56 printprocesslog "INFO rsync files with "$bwlimit
57 echo "rsync files with "$bwlimit >> $logfile 2>&1
58 searchwindow="-cmin -840"
59 printprocesslog "INFO treat the last 14 hours."
60 echo "INFO treat the last 14 hours." >> $logfile 2>&1
61else
62 # no bw limit during day
63 bwlimit=""
64 printprocesslog "INFO rsync files without bwlimit"
65 echo "rsync files without bwlimit" >> $logfile 2>&1
66 searchwindow="-ctime -6"
67 printprocesslog "INFO treat the last 14 hours."
68 echo "INFO treat the last 14 hours." >> $logfile 2>&1
69fi
70
71# find directories to be rsynced
72dirs=( `ssh newdaq "find /loc_data/aux/ -mindepth 3 $searchwindow -type d | sort "` )
73
74# do rsync for auxfiles in these directories
75for dir in ${dirs[@]}
76do
77 echo "" >> $logfile 2>&1
78 printprocesslog "INFO processing files in "$dir
79 echo `date`": processing files in "$dir >> $logfile 2>&1
80 #echo `date`": processing files in "$dir
81
82 # first take care of SSD /scratch
83 scratch=`echo $dir | sed -e 's/loc_data/scratch/'`
84 if ! [ -d $scratch ]
85 then
86 mkdir -pv $scratch >> $logfile 2>&1
87 fi
88 # rsync from newdaq to newdata
89 if ! /usr/bin/rsync -avu -T $rsynctempdir1 $bwlimit newdaq:$dir/ $scratch >> $logfile 2>&1
90 then
91 printprocesslog "CONNECTION problem rsyncing auxiliary files in "$dir" from newdaq to newdata"
92 echo `date`": problem rsyncing auxiliary files in "$dir" from newdaq to newdata" >> $logfile 2>&1
93 fi
94
95 # now make copies on newdata
96 data1=`echo $dir | sed -e 's/loc_data/data1/'`
97 if ! [ -d $data1 ]
98 then
99 mkdir -pv $data1 >> $logfile 2>&1
100 fi
101 data2=`echo $dir | sed -e 's/loc_data/data2/'`
102 if ! [ -d $data2 ]
103 then
104 mkdir -pv $data2 >> $logfile 2>&1
105 fi
106 # distribute to several disk on newdata
107 if ! /usr/bin/rsync -avu -T $rsynctempdir2 $bwlimit $scratch/ $data1 >> $logfile 2>&1
108 then
109 printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data1"
110 echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data1" >> $logfile 2>&1
111 fi
112 if ! /usr/bin/rsync -avu -T $rsynctempdir2 $bwlimit $scratch/ $data2 >> $logfile 2>&1
113 then
114 printprocesslog "WARN problem rsyncing auxiliary files in "$dir" from scratch to data2"
115 echo `date`": problem rsyncing auxiliary files in "$dir" from scratch to data2" >> $logfile 2>&1
116 fi
117done
118
119finish
Note: See TracBrowser for help on using the repository browser.