Changeset 14840 for trunk/DataCheck/Transfer/ZipRawData.sh

Timestamp:

02/04/13 06:02:39 (12 years ago)

Author:

Daniela Dorner

Message:

adapted to new structure in LP (newdaq, new transfer scheme)

File:

• trunk/DataCheck/Transfer/ZipRawData.sh (modified) (6 diffs)

trunk/DataCheck/Transfer/ZipRawData.sh

@@ -10 +10 @@
 date >> $logfile
 
+if ! ls /daq/raw >/dev/null 2>&1
+then 
+   printprocesslog "ERROR /daq/raw is not available."
+   finish
+fi
 # get last 3, 6 or 9 nights
 dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \
@@ -23 +28 @@
         )
 
-# time rawfile is untouched before further processing (i.e. adding checksum and zipping)
-delaytime=30 #30 minutes
-
 # setup to use ftools
 source $HEADAS/headas-init.sh
+
+delaytime=5
 
 # do rsync for rawfiles of these dates
@@ -33 +37 @@
 do 
    echo "" >> $logfile 2>&1
-   rawdir=/loc_data/raw/$date
+   rawdir=/daq/raw/$date
    echo `date`": processing files in "$rawdir >> $logfile 2>&1
    printprocesslog "INFO processing files in "$rawdir
@@ -69 +73 @@
    do
       # filename for temporary and final zipfile
-      zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-      zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+      zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+      zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
       # check if zipped file already exists
       if [ -e $zipfile ]
@@ -86 +90 @@
       fi
 
-      # check if file is already finished
-      # original file on daq (if data was taken on daq
-      origfile=`echo $file | sed -e 's/loc_data/daq/'`
-      if [ -e $origfile ]
-      then
-         # get time of last modification as seconds since Epoch for both files
-         timeorig=`stat -c %Y $origfile`
-         timecopy=`stat -c %Y $file`
-         # compare times
-         if ! [ $timeorig -eq $timecopy ]
-         then
-            # if times are not the same, the file is still open => no zip
-            echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1
-            printprocesslog "WARN file "$file" not yet closed => continue"
-            continue
-         fi
-      else
-         # if the origfile doesn't exist, the data was probably written not on daq but on data
-         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1
-         printprocesslog "WARN file "$file" was probably taken on data and not daq "
-      fi
+# not needed anymore, as only complete files are transferred from newdaq to daq
+#      # check if file is already finished
+#      # original file on daq (if data was taken on daq
+#      origfile=`echo $file | sed -e 's/loc_data/daq/'`
+#      if [ -e $origfile ]
+#      then
+#         # get time of last modification as seconds since Epoch for both files
+#         timeorig=`stat -c %Y $origfile`
+#         timecopy=`stat -c %Y $file`
+#         # compare times
+#         if ! [ $timeorig -eq $timecopy ]
+#         then
+#            # if times are not the same, the file is still open => no zip
+#            echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1
+#            printprocesslog "WARN file "$file" not yet closed => continue"
+#            continue
+#         fi
+#      else
+#         # if the origfile doesn't exist, the data was probably written not on daq but on data
+#         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1
+#         printprocesslog "WARN file "$file" was probably taken on data and not daq "
+#      fi
 
 #      # update the raw file with the checksums
@@ -133 +138 @@
          printprocesslog "INFO remove "$zipfiletmp"..."
          rm -v $zipfiletmp >> $logfile 2>&1
+         diskusage=( `df -P /loc_data | grep /loc_data ` )
+         # check if more than ~ 10 GB are left on /loc_data
+         if [ ${diskusage[3]} -lt 10000000 ] 
+         then
+            echo "WARN less than 10 GB left on /loc_data on data "
+            printprocesslog "WARN less than 10 GB left on /loc_data on data "
+            finish
+         fi
       fi
    done