Changeset 12575 for trunk/DataCheck/ZipRawData.sh

Timestamp:

11/20/11 07:11:38 (13 years ago)

Author:

Daniela Dorner

Message:

included algorithm to make sure that only finished files are zipped

File:

• trunk/DataCheck/ZipRawData.sh (modified) (3 diffs)

trunk/DataCheck/ZipRawData.sh

@@ -7 +7 @@
 logfile=/home/`whoami`/DataCheck/log/ZipRaw$today.log
 
+echo "" >> $logfile 2>&1
+echo "" >> $logfile 2>&1
+echo "" >> $logfile 2>&1
 rawdir=/loc_data/raw/$year/$month
 echo `date`": processing files in "$rawdir >> $logfile
@@ -26 +29 @@
 done
 
+# find all fits-files starting with the oldest file
 echo `date`": finding files to be zipped in $rawdir..." >> $logfile
-fitsfiles=`find $rawdir -type f -name '*fits' | sort -r`
+fitsfiles=`find $rawdir -type f -name '*fits'| sort `
 
+# loop to zip files
 echo `date`": zipping files in $rawdir..." >> $logfile
 for file in $fitsfiles 
 do 
+   # filename for temporary and final zipfile
    zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` 
    zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'` 
@@ -39 +45 @@
       continue
    fi
+
+   # check if files was accessed in the last 30 minutes
+   isnew=`find $file -amin -30`
+   if [ "$isnew" != "" ]
+   then
+      echo $file" is not older than 30 min => continue" >> $logfile
+      continue
+   fi
+
+   # check if file is already finished
+   # original file on daq (if data was taken on daq
+   origfile=`echo $file | sed -e 's/loc_data/daq/'` 
+   if [ -e $origfile ]
+   then
+      # get time of last modification as seconds since Epoch for both files
+      timeorig=`stat -c %Y $origfile`
+      timecopy=`stat -c %Y $file`
+      # compare times
+      if ! [ $timeorig -eq $timecopy ]
+      then
+         # if times are not the same, the file is still open => no zip
+         echo `date`": file "$file" not yet closed => continue" >> $logfile
+         continue
+      fi
+   else
+      # if the origfile doesn't exist, the data was probably written not on daq but on data
+      echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
+   fi
+
    echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile
    # zip file to stdout and pipe it to outputfile
    if pigz -1 -c -f $file > $zipfiletmp;  
    then
+      # if successful, move temporary to final zipfile
       mv -v $zipfiletmp $zipfile >> $logfile
    else
+      # if not successful, remove temporary zipfile
       rm -v $zipfiletmp >> $logfile
    fi
 done
+
+