Changeset 17590 for trunk/DataCheck/Transfer

Timestamp:

03/02/14 00:35:31 (11 years ago)

Author:

Daniela Dorner

Message:

added copying of already compressed files

File:

• trunk/DataCheck/Transfer/ZipRawData.sh (modified) (5 diffs)

trunk/DataCheck/Transfer/ZipRawData.sh

@@ -27 +27 @@
 fi
 
-# setup to use ftools
-source $HEADAS/headas-init.sh
+# not needed anymore as ftools are not used anymore
+## setup to use ftools
+#source $HEADAS/headas-init.sh
 
+# files younger than $delaytime are not processed
 delaytime=5
 
@@ -47 +49 @@
       continue
    fi
-   zipdir=/loc_data/zipraw/$date
-   if ! [ -d $zipdir ]
-   then
-      # create output directory for zip
-      makedir $zipdir >> $logfile 2>&1
-   fi
 
    # find all fits-files starting with the oldest file
    echo `date`": finding files to be zipped in $rawdir..." >> $logfile 2>&1
    printprocesslog "INFO finding files to be zipped in "$rawdir"..."
-   fitsfiles=`find $rawdir -type f -name '*fits'| sort `
+   fitsfiles=`find $rawdir -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort `
 
    if [ ${#fitsfiles[@]} -eq 0 ]
@@ -66 +62 @@
    fi
 
+   zipdir=/loc_data/zipraw/$date
+   # create output directory for zip
+   makedir $zipdir >> $logfile 2>&1
+
    # loop to zip files
    echo `date`": zipping files in $rawdir..." >> $logfile 2>&1
@@ -71 +71 @@
    for file in $fitsfiles
    do
-      # filename for temporary and final zipfile
-      zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
-      zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
-      # check if zipped file already exists
-      if [ -e $zipfile ]
+      # check if more than ~ 10 GB are left on /loc_data
+      diskusage=( `df -P /loc_data | grep /loc_data ` )
+      if [ ${diskusage[3]} -lt 10000000 ] 
       then
-         continue
+         echo "WARN less than 10 GB left on /loc_data on data " >> $logfile 2>&1
+         printprocesslog "WARN less than 10 GB left on /loc_data on data "
+         finish
       fi
-
+      
       # check if raw file was accessed in the last $delaytime minutes
       isnew=`find $file -amin -$delaytime`
@@ -89 +89 @@
       fi
 
-# not needed anymore, as only complete files are transferred from newdaq to daq
-#      # check if file is already finished
-#      # original file on daq (if data was taken on daq
-#      origfile=`echo $file | sed -e 's/loc_data/daq/'`
-#      if [ -e $origfile ]
-#      then
-#         # get time of last modification as seconds since Epoch for both files
-#         timeorig=`stat -c %Y $origfile`
-#         timecopy=`stat -c %Y $file`
-#         # compare times
-#         if ! [ $timeorig -eq $timecopy ]
-#         then
-#            # if times are not the same, the file is still open => no zip
-#            echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1
-#            printprocesslog "WARN file "$file" not yet closed => continue"
-#            continue
-#         fi
-#      else
-#         # if the origfile doesn't exist, the data was probably written not on daq but on data
-#         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1
-#         printprocesslog "WARN file "$file" was probably taken on data and not daq "
-#      fi
+      iszipped=`echo $file | grep -o fz`
+      if [ "$iszipped" == "fz" ]
+      then
+         # treat already compressed files
+         printprocesslog "DEBUG treat compressed file "$file
 
-#      # update the raw file with the checksums
-#      ftchecksum update=yes $file >> $logfile 2>&1
-#      check=$?
-#      if [ $check -eq 0 ]
-#      then
-#         echo `date`": file "$file" was updated with the checksums " >> $logfile 2>&1
-#      else
-#         echo `date`": problem when updating file "$file" (ftchecksum exited with "$check")" >> $logfile 2>&1
-#         echo `date`": problem when updating file "$file" (ftchecksum exited with "$check")"
-#         continue
-#      fi
+         # filename for file on data
+         zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' `
+         zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
+         # check if file on data already exists
+         if [ -e $zipfile ]
+         then
+            continue
+         fi
 
-      echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile 2>&1
-      printprocesslog "INFO zipping "$file" to "$zipfile" ..."
-      #echo "INFO zipping "$file" to "$zipfile" ..."
-      # read setup again to allow for updates of variables defining transfer
-      source `dirname $0`/../Sourcefile.sh
-      # zip file to stdout and pipe it to outputfile
-      echo "pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp" >> $logfile 2>&1
-      pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp
-      statuspigz=( `echo ${PIPESTATUS[@]}` )
-      printprocesslog "DEBUG PIPESTATUS:"${statuspigz[@]}
-      echo " PIPESTATUS:"${statuspigz[@]} >> $logfile 2>&1
-      if [ ${statuspigz[0]} -eq 0 ] && [ ${statuspigz[1]} -eq 0 ]
-      then
-         # if successful, move temporary to final zipfile
-         printprocesslog "INFO move "$zipfiletmp" to "$zipfile" ..."
-         mv -v $zipfiletmp $zipfile >> $logfile 2>&1
+         echo `date`": copying "$file" to "$zipfile" ..." >> $logfile 2>&1
+         printprocesslog "INFO copying "$file" to "$zipfile" ..."
+         #echo "INFO zipping "$file" to "$zipfile" ..."
+         # read setup again to allow for updates of variables defining transfer
+         source `dirname $0`/../Sourcefile.sh
+         # zip file to stdout and pipe it to outputfile
+         echo "pv --rate-limit $limitpigz $file > $zipfiletmp" >> $logfile 2>&1
+         pv --rate-limit $limitpigz $file > $zipfiletmp
+         statuspv=$?
+         printprocesslog "DEBUG pvstatus:"$statuspv
+         echo " pvstatus:"$statuspv >> $logfile 2>&1
+         if [ $statuspv -eq 0 ] 
+         then
+            # if successful, move temporary to final zipfile
+            printprocesslog "INFO move "$zipfiletmp" to "$zipfile" ..."
+            mv -v $zipfiletmp $zipfile >> $logfile 2>&1
+         else
+            # if not successful, remove temporary zipfile
+            printprocesslog "WARN pv failed (statuspv:"$statuspv")."
+            echo "WARN pv failed (PIPESTATUS:"$statuspv")." >> $logfile 2>&1
+            printprocesslog "INFO remove "$zipfiletmp"..."
+            rm -v $zipfiletmp >> $logfile 2>&1
+            finish
+         fi
       else
-         # if not successful, remove temporary zipfile
-         printprocesslog "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")."
-         echo "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")." >> $logfile 2>&1
-         finish
-         printprocesslog "INFO remove "$zipfiletmp"..."
-         rm -v $zipfiletmp >> $logfile 2>&1
-         diskusage=( `df -P /loc_data | grep /loc_data ` )
-         # check if more than ~ 10 GB are left on /loc_data
-         if [ ${diskusage[3]} -lt 10000000 ] 
+         # treat uncompressed files
+         printprocesslog "DEBUG treat uncompressed file "$file
+
+         # filename for temporary and final zipfile
+         zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+         zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+         # check if zipped file already exists
+         if [ -e $zipfile ]
          then
-            echo "WARN less than 10 GB left on /loc_data on data " >> $logfile 2>&1
-            printprocesslog "WARN less than 10 GB left on /loc_data on data "
+            continue
+         fi
+
+         echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile 2>&1
+         printprocesslog "INFO zipping "$file" to "$zipfile" ..."
+         #echo "INFO zipping "$file" to "$zipfile" ..."
+         # read setup again to allow for updates of variables defining transfer
+         source `dirname $0`/../Sourcefile.sh
+         # zip file to stdout and pipe it to outputfile
+         echo "pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp" >> $logfile 2>&1
+         pv --rate-limit $limitpigz $file | pigz -1 -c -f -p $numprocpigz > $zipfiletmp
+         statuspigz=( `echo ${PIPESTATUS[@]}` )
+         printprocesslog "DEBUG PIPESTATUS:"${statuspigz[@]}
+         echo " PIPESTATUS:"${statuspigz[@]} >> $logfile 2>&1
+         if [ ${statuspigz[0]} -eq 0 ] && [ ${statuspigz[1]} -eq 0 ]
+         then
+            # if successful, move temporary to final zipfile
+            printprocesslog "INFO move "$zipfiletmp" to "$zipfile" ..."
+            mv -v $zipfiletmp $zipfile >> $logfile 2>&1
+         else
+            # if not successful, remove temporary zipfile
+            printprocesslog "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")."
+            echo "WARN pigz or pv failed (PIPESTATUS:"${statuspigz[@]}")." >> $logfile 2>&1
+            printprocesslog "INFO remove "$zipfiletmp"..."
+            rm -v $zipfiletmp >> $logfile 2>&1
             finish
          fi