Changeset 12582 for trunk/DataCheck/ZipRawData.sh

Timestamp:

11/21/11 06:56:22 (13 years ago)

Author:

Daniela Dorner

Message:

changed to take into account the current and the 3 previous nights

File:

• trunk/DataCheck/ZipRawData.sh (modified) (1 diff)

trunk/DataCheck/ZipRawData.sh

@@ -1 +1 @@
 #!/bin/bash
-
-year=`date +%Y --date="-1day"`
-month=`date +%m --date="-1day"`
 
 today=`date +%F`
 logfile=/home/`whoami`/DataCheck/log/ZipRaw$today.log
 
-echo "" >> $logfile 2>&1
-echo "" >> $logfile 2>&1
-echo "" >> $logfile 2>&1
-rawdir=/loc_data/raw/$year/$month
-echo `date`": processing files in "$rawdir >> $logfile
+# get last 3 nights
+dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
 
-#find all directories in $rawdir
-dirs=`find $rawdir -type d | sort`
-
-echo `date`": create missing directories in /loc_data/zipraw/$year/$month" >> $logfile
-for dir in $dirs 
+# do rsync for rawfiles of these dates
+for date in ${dates[@]}
 do 
-   zipdir=`echo $dir | sed -e 's/raw/zipraw/'` 
-   # check if output directory for zip already exists
-   if [ -d $zipdir ] 
-   then 
+   echo "" >> $logfile 2>&1
+   echo "" >> $logfile 2>&1
+   echo "" >> $logfile 2>&1
+   rawdir=/loc_data/raw/$date
+   echo `date`": processing files in "$rawdir >> $logfile
+   # check if data are available from that night
+   if ! [ -d $rawdir ]
+   then
+      echo `date`": no data available in "$rawdir >> $logfile
       continue
    fi
-   # create output directory for zip
-   mkdir -pv $zipdir >> $logfile
-done
-
-# find all fits-files starting with the oldest file
-echo `date`": finding files to be zipped in $rawdir..." >> $logfile
-fitsfiles=`find $rawdir -type f -name '*fits'| sort `
-
-# loop to zip files
-echo `date`": zipping files in $rawdir..." >> $logfile
-for file in $fitsfiles 
-do 
-   # filename for temporary and final zipfile
-   zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` 
-   zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'` 
-   # check if zipped file already exists
-   if [ -e $zipfile ] 
-   then 
-      continue
+   zipdir=/loc_data/zipraw/$date
+   if ! [ -d $zipdir ]
+   then
+      # create output directory for zip
+      mkdir -pv $zipdir >> $logfile
    fi
 
-   # check if files was accessed in the last 30 minutes
-   isnew=`find $file -amin -30`
-   if [ "$isnew" != "" ]
-   then
-      echo $file" is not older than 30 min => continue" >> $logfile
-      continue
-   fi
+   # find all fits-files starting with the oldest file
+   echo `date`": finding files to be zipped in $rawdir..." >> $logfile
+   fitsfiles=`find $rawdir -type f -name '*fits'| sort `
 
-   # check if file is already finished
-   # original file on daq (if data was taken on daq
-   origfile=`echo $file | sed -e 's/loc_data/daq/'` 
-   if [ -e $origfile ]
-   then
-      # get time of last modification as seconds since Epoch for both files
-      timeorig=`stat -c %Y $origfile`
-      timecopy=`stat -c %Y $file`
-      # compare times
-      if ! [ $timeorig -eq $timecopy ]
+   # loop to zip files
+   echo `date`": zipping files in $rawdir..." >> $logfile
+   for file in $fitsfiles
+   do
+      # filename for temporary and final zipfile
+      zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
+      zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
+      # check if zipped file already exists
+      if [ -e $zipfile ]
       then
-         # if times are not the same, the file is still open => no zip
-         echo `date`": file "$file" not yet closed => continue" >> $logfile
          continue
       fi
-   else
-      # if the origfile doesn't exist, the data was probably written not on daq but on data
-      echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
-   fi
 
-   echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile
-   # zip file to stdout and pipe it to outputfile
-   if pigz -1 -c -f $file > $zipfiletmp;  
-   then
-      # if successful, move temporary to final zipfile
-      mv -v $zipfiletmp $zipfile >> $logfile
-   else
-      # if not successful, remove temporary zipfile
-      rm -v $zipfiletmp >> $logfile
-   fi
+      # check if raw file was accessed in the last 30 minutes
+      isnew=`find $file -amin -30`
+      if [ "$isnew" != "" ]
+      then
+         echo $file" is not older than 30 min => continue" >> $logfile
+         continue
+      fi
+
+      # check if file is already finished
+      # original file on daq (if data was taken on daq
+      origfile=`echo $file | sed -e 's/loc_data/daq/'`
+      if [ -e $origfile ]
+      then
+         # get time of last modification as seconds since Epoch for both files
+         timeorig=`stat -c %Y $origfile`
+         timecopy=`stat -c %Y $file`
+         # compare times
+         if ! [ $timeorig -eq $timecopy ]
+         then
+            # if times are not the same, the file is still open => no zip
+            echo `date`": file "$file" not yet closed => continue" >> $logfile
+            continue
+         fi
+      else
+         # if the origfile doesn't exist, the data was probably written not on daq but on data
+         echo `date`": file "$file" was probably taken on data and not daq " >> $logfile
+      fi
+
+      echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile
+      # zip file to stdout and pipe it to outputfile
+      if pigz -1 -c -f $file > $zipfiletmp;
+      then
+         # if successful, move temporary to final zipfile
+         mv -v $zipfiletmp $zipfile >> $logfile
+      else
+         # if not successful, remove temporary zipfile
+         rm -v $zipfiletmp >> $logfile
+      fi
+   done
 done