Changeset 18872 for trunk/DataCheck/Transfer

Timestamp:

05/21/17 10:34:57 (7 years ago)

Author:

Daniela Dorner

Message:

implemented option to use /data1 instead of /scratch

File:

• trunk/DataCheck/Transfer/ZipRawData.sh (modified) (6 diffs)

trunk/DataCheck/Transfer/ZipRawData.sh

@@ -1 +1 @@
 #!/bin/bash
 
-# this script has been written to run on La Palma on the machine data
+# this script has been written to run on La Palma on the machine newdata
 #   i.e. paths are only working on this machine
 
+# to use script with /data1 instead of /scratch 
+#  (e.g. when /scratch is full) 
+# export data1=yes
+# before executing the script
 source `dirname $0`/../Sourcefile.sh
 printprocesslog "INFO starting "$0
@@ -27 +31 @@
 fi
 
+rsynctempdir=/data1/rsync_tmp
+if ! [ -d $rsynctempdir ]
+then
+   mkdir $rsynctempdir
+fi
+
 # not needed anymore as ftools are not used anymore
 ## setup to use ftools
@@ -38 +48 @@
 do 
    echo "" >> $logfile 2>&1
-   rawdir=/scratch/raw/$date
+   if [ "$data1" = "yes" ]
+   then 
+      rawdir=/data1/raw/$date
+      printprocesslog "INFO using "$rawdir" as input"
+   else 
+      rawdir=/scratch/raw/$date
+   fi
    echo `date`": processing files in "$rawdir >> $logfile 2>&1
    printprocesslog "INFO processing files in "$rawdir
@@ -98 +114 @@
       fi
       
+      printprocesslog "INFO process file "$file
+      if [ "$certaindate" != "" ]
+      then
+         echo "INFO process file "$file
+      fi
       # first rsync from /scratch to /data1
-      file2=`echo $file | sed -e 's/scratch/data1/'`
-      if ! rsync -au -T $rsynctempdir $file $file2
-      then
-         printprocesslog "ERROR something went wrong with rsync of "$file
-         rm $file2
-         continue
-      fi
-      
+      if [ "$data1" != "yes" ]
+      then 
+         file2=`echo $file | sed -e 's/scratch/data1/'`
+         if ! rsync -au -T $rsynctempdir $file $file2
+         then
+            printprocesslog "ERROR something went wrong with rsync of "$file
+            rm $file2
+            continue
+         fi
+      fi
+      # copying to /data2
       iszipped=`echo $file | grep -o fz`
       if [ "$iszipped" == "fz" ]
@@ -114 +138 @@
 
          # filename for file on data
-         zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' `
-         zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
-         
+         if [ "$data1" = "yes" ]
+         then 
+            zipfile=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' `
+            zipfiletmp=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
+         else
+            zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' `
+            zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
+         fi
          # check if file on data already exists
          if [ -e $zipfile ]
@@ -152 +181 @@
 
          # filename for temporary and final zipfile
-         zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-         zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+         if [ "$data1" = "yes" ]
+         then 
+            zipfile=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+            zipfiletmp=`echo $file | sed -e 's/data1/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+         else
+            zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+            zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+         fi
          
          # check if zipped file already exists