Changeset 18853 for trunk/DataCheck/Transfer

Timestamp:

05/14/17 20:58:29 (7 years ago)

Author:

Daniela Dorner

Message:

updated to run on newdata and create a second copy

File:

• trunk/DataCheck/Transfer/ZipRawData.sh (modified) (8 diffs)

trunk/DataCheck/Transfer/ZipRawData.sh

@@ -10 +10 @@
 date >> $logfile
 
-if ! ls /daq/raw >/dev/null 2>&1
+if ! ls /scratch/raw >/dev/null 2>&1
 then 
-   printprocesslog "ERROR /daq/raw is not available."
+   printprocesslog "ERROR /scratch/raw is not available."
    finish
 fi
-# if daq unavailable
-#if ! ls /newdaq/raw >/dev/null 2>&1
-#then 
-#   printprocesslog "ERROR /newdaq/raw is not available."
-#   finish
-#fi
 
 # get dates
@@ -37 +31 @@
 #source $HEADAS/headas-init.sh
 
-# files younger than $delaytime are not processed
+# files younger than $delaytime minutes are not processed
 delaytime=5
 
@@ -44 +38 @@
 do 
    echo "" >> $logfile 2>&1
-   rawdir=/daq/raw/$date
-#   rawdir=/newdaq/raw/$date #if daq unavailable
+   rawdir=/scratch/raw/$date
    echo `date`": processing files in "$rawdir >> $logfile 2>&1
    printprocesslog "INFO processing files in "$rawdir
@@ -68 +61 @@
       continue
    fi
-
-   zipdir=/loc_data/zipraw/$date
-   # create output directory for zip
+   
+   rawdir2=/data1/raw/$date
+   zipdir=/data2/zipraw/$date
+   # create output directories
+   makedir $rawdir2 >> $logfile 2>&1
    makedir $zipdir >> $logfile 2>&1
 
@@ -78 +73 @@
    for file in $fitsfiles
    do
-      # check if more than ~ 10 GB are left on /loc_data
-      diskusage=( `df -P /loc_data | grep /loc_data ` )
+      # check if more than ~ 10 GB are left on output directories
+      diskusage=( `df -P /data1 | grep /data1 ` )
       if [ ${diskusage[3]} -lt 10000000 ] 
       then
-         echo "WARN less than 10 GB left on /loc_data on data ("${diskusage[3]}")" >> $logfile 2>&1
-         printprocesslog "WARN less than 10 GB left on /loc_data on data ("${diskusage[3]}")"
+         echo "WARN less than 10 GB left on /data1 on newdata ("${diskusage[3]}")" >> $logfile 2>&1
+         printprocesslog "WARN less than 10 GB left on /data1 on newdata ("${diskusage[3]}")"
+         finish
+      fi
+      diskusage=( `df -P /data2 | grep /data2 ` )
+      if [ ${diskusage[3]} -lt 10000000 ] 
+      then
+         echo "WARN less than 10 GB left on /data2 on newdata ("${diskusage[3]}")" >> $logfile 2>&1
+         printprocesslog "WARN less than 10 GB left on /data2 on newdata ("${diskusage[3]}")"
          finish
       fi
@@ -95 +97 @@
          continue
       fi
-
+      
+      # first rsync from /scratch to /data1
+      file2=`echo $file | sed -e 's/scratch/data1/'`
+      if ! rsync -au -T $rsynctempdir $file $file2
+      then
+         printprocesslog "ERROR something went wrong with rsync of "$file
+         rm $file2
+         continue
+      fi
+      
       iszipped=`echo $file | grep -o fz`
       if [ "$iszipped" == "fz" ]
@@ -103 +114 @@
 
          # filename for file on data
-         zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' `
-         zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
-#         # if daq unavailable
-#         zipfile=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' `
-#         zipfiletmp=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
+         zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' `
+         zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
          
          # check if file on data already exists
@@ -144 +152 @@
 
          # filename for temporary and final zipfile
-         zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-         zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
-#         # if daq unavailable
-#         zipfile=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-#         zipfiletmp=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+         zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+         zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
          
          # check if zipped file already exists