Changeset 15517 for trunk/DataCheck/Transfer

Timestamp:

05/07/13 23:22:58 (12 years ago)

Author:

Daniela Dorner

Message:

added pv to control rate of zipping, limited pigz to 2 processes

File:

• trunk/DataCheck/Transfer/ZipRawData.sh (modified) (4 diffs)

trunk/DataCheck/Transfer/ZipRawData.sh

@@ -15 +15 @@
    finish
 fi
-# get last 3, 6 or 9 nights
-dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \
-#        `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \
-#        `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \
-        )
-
-# get last 3, 6 or 9 nights
-dates=( 
-        `date +%Y/%m/%d --date="-204hour"`  `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-156hour"` \
-        `date +%Y/%m/%d --date="-132hour"`  `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-84hour"`  \
-        `date +%Y/%m/%d --date="-60hour"`   `date +%Y/%m/%d --date="-36hour"`  `date +%Y/%m/%d --date="-12hour"`  \
-        )
+# get current hour
+hour=`date +%k`
+if [ $hour -le 7 ] || [ $hour -ge 19 ]
+then
+   dates=( `date +%Y/%m/%d --date="-12hour"` )
+else
+   # get last 3, 6 or 9 nights
+   dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \
+   #        `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \
+   #        `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \
+           )
+fi
 
 # setup to use ftools
@@ -128 +128 @@
       printprocesslog "INFO zipping "$file" to "$zipfile" ..."
       #echo "INFO zipping "$file" to "$zipfile" ..."
+      source `dirname $0`/../Sourcefile.sh
       # zip file to stdout and pipe it to outputfile
-      if pigz -1 -c -f $file > $zipfiletmp
+      echo "pv --rate-limit $limitpigz $file | pigz -1 -c -f -p 2 > $zipfiletmp" >> $logfile 2>&1
+      pv --rate-limit $limitpigz $file | pigz -1 -c -f -p 2 > $zipfiletmp
+      statuspigz=( `echo ${PIPESTATUS[@]}` )
+      printprocesslog "DEBUG PIPESTATUS:"${statuspigz[@]}
+      echo " PIPESTATUS:"${statuspigz[@]} >> $logfile 2>&1
+      if [ ${statuspigz[0]} -eq 0 ] && [ ${statuspigz[1]} -eq 0 ]
       then
          # if successful, move temporary to final zipfile
@@ -136 +142 @@
       else
          # if not successful, remove temporary zipfile
+         printprocesslog "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")."
+         echo "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")." >> $logfile 2>&1
+         finish
          printprocesslog "INFO remove "$zipfiletmp"..."
          rm -v $zipfiletmp >> $logfile 2>&1
@@ -142 +151 @@
          if [ ${diskusage[3]} -lt 10000000 ] 
          then
-            echo "WARN less than 10 GB left on /loc_data on data "
+            echo "WARN less than 10 GB left on /loc_data on data " >> $logfile 2>&1
             printprocesslog "WARN less than 10 GB left on /loc_data on data "
             finish