Changeset 18869
- Timestamp:
- 05/21/17 10:32:05 (8 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/DataCheck/QuickLook/Step1.sh
r18855 r18869 1 1 #!/bin/bash 2 2 # 3 4 # to use script with /data1 instead of /scratch 5 # (e.g. when /scratch is full) 6 # export data1=yes 7 # before executing the script 8 # preferably ZipRawData.sh has processed at this point all 9 # files available already on /scratch 3 10 4 11 # remarks: … … 36 43 makedir $calpath >/dev/null 37 44 rawpathnewdaq=/newdaq/raw/$datepath 38 rawpath=/loc_data/raw/$datepath 45 if [ "$data1" = "yes" ] 46 then 47 rawpath=/data1/raw/$datepath 48 rsynctempdir=/data1/rsync_tmp 49 printprocesslog "INFO using "$rawpath" for processing" 50 else 51 rawpath=/scratch/raw/$datepath 52 rsynctempdir=/scratch/rsync_tmp 53 fi 54 if ! [ -d $rsynctempdir ] 55 then 56 mkdir $rsynctempdir 57 fi 58 59 39 60 40 61 # needed auxiliary files: … … 66 87 function check_disks() 67 88 { 68 # put here the check for /scratch (output of rsync) and /data1 (output of analysis) 89 # at least 5% free disk on /data1 90 diskusage=( `df -P /data1 | grep data1 ` ) 91 if [ ${diskusage[3]} -lt $disklimitnewdata2 ] 92 then 93 echo "WARN less than 5% left on /data1 on node "$HOSTNAME 94 printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME 95 df -h /data1 96 finish 97 fi 98 99 # if /data1 is used for processing, /scratch doesn't need to be checked 100 if [ "$data1" = "yes" ] 101 then 102 return 103 fi 69 104 70 105 # at least 10% free disk on /scratch … … 77 112 finish 78 113 fi 79 # at least 5% free disk on /data180 diskusage=( `df -P /data1 | grep data1 ` )81 if [ ${diskusage[3]} -lt $disklimitnewdata2 ]82 then83 echo "WARN less than 5% left on /data1 on node "$HOSTNAME84 printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME85 df -h /scratch86 finish87 fi88 114 } 89 115 … … 92 118 # getting lists of files 93 119 printprocesslog "INFO get lists of raw files on newdaq and daq" 94 files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) 120 #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) 121 files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` ) 95 122 if [ ${#files[@]} -eq 0 ] 96 123 then … … 125 152 # when there is more than 10% space on daq 126 153 source `dirname $0`/../Sourcefile.sh 127 check_disks128 154 129 155 numcalibrated=0 … … 137 163 for file in ${files[@]} 138 164 do 165 # check if still enough diskspace for transfer 166 check_disks 167 if [ "$certaindate" != "" ] 168 then 169 echo "processing "$file 170 fi 139 171 printprocesslog "processing "$file 140 echo "processing "$file 141 #localfile=`echo $file | sed -e 's/newdaq/scratch/'` 142 localfile=$file 172 if [ "$data1" = "yes" ] 173 then 174 localfile=`echo $file | sed -e 's/newdaq/data1/'` 175 else 176 localfile=`echo $file | sed -e 's/newdaq/scratch/'` 177 fi 178 #localfile=$file 143 179 144 180 source `dirname $0`/../Sourcefile.sh … … 176 212 177 213 # rsync 178 if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile 214 #if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile 215 if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit $file $localfile 179 216 then 180 217 printprocesslog "ERROR something went wrong with rsync of "$file … … 231 268 232 269 # get run number 233 runnum=` echo $localfile | cut -d_ -f3| cut -d. -f1`270 runnum=`basename $localfile | cut -d_ -f2 | cut -d. -f1` 234 271 235 272 # what is needed to process the different runs? … … 351 388 # get new file lists 352 389 printprocesslog "INFO get new file lists for "$datepath 353 files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) 390 #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) 391 files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` ) 354 392 fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) 355 393 callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` ) … … 364 402 sleep 60 365 403 printprocesslog "INFO get new file lists for "$datepath 366 files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) 404 #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) 405 files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` ) 367 406 fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) 368 407 callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
Note:
See TracChangeset
for help on using the changeset viewer.