| 1 | #!/bin/bash
|
|---|
| 2 |
|
|---|
| 3 | today=`date +%F`
|
|---|
| 4 | logfile=/home/`whoami`/DataCheck/log/ZipRaw$today.log
|
|---|
| 5 |
|
|---|
| 6 | # get last 3 nights
|
|---|
| 7 | dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
|
|---|
| 8 |
|
|---|
| 9 | # do rsync for rawfiles of these dates
|
|---|
| 10 | for date in ${dates[@]}
|
|---|
| 11 | do
|
|---|
| 12 | echo "" >> $logfile 2>&1
|
|---|
| 13 | echo "" >> $logfile 2>&1
|
|---|
| 14 | echo "" >> $logfile 2>&1
|
|---|
| 15 | rawdir=/loc_data/raw/$date
|
|---|
| 16 | echo `date`": processing files in "$rawdir >> $logfile 2>&1
|
|---|
| 17 | # check if data are available from that night
|
|---|
| 18 | if ! [ -d $rawdir ]
|
|---|
| 19 | then
|
|---|
| 20 | echo `date`": no data available in "$rawdir >> $logfile 2>&1
|
|---|
| 21 | continue
|
|---|
| 22 | fi
|
|---|
| 23 | zipdir=/loc_data/zipraw/$date
|
|---|
| 24 | if ! [ -d $zipdir ]
|
|---|
| 25 | then
|
|---|
| 26 | # create output directory for zip
|
|---|
| 27 | mkdir -pv $zipdir >> $logfile 2>&1
|
|---|
| 28 | fi
|
|---|
| 29 |
|
|---|
| 30 | # find all fits-files starting with the oldest file
|
|---|
| 31 | echo `date`": finding files to be zipped in $rawdir..." >> $logfile 2>&1
|
|---|
| 32 | fitsfiles=`find $rawdir -type f -name '*fits'| sort `
|
|---|
| 33 |
|
|---|
| 34 | if [ ${#fitsfiles[@]} -eq 0 ]
|
|---|
| 35 | then
|
|---|
| 36 | echo `date`": no files to be zipped in $rawdir..." >> $logfile 2>&1
|
|---|
| 37 | continue
|
|---|
| 38 | fi
|
|---|
| 39 |
|
|---|
| 40 | # loop to zip files
|
|---|
| 41 | echo `date`": zipping files in $rawdir..." >> $logfile 2>&1
|
|---|
| 42 | for file in $fitsfiles
|
|---|
| 43 | do
|
|---|
| 44 | # filename for temporary and final zipfile
|
|---|
| 45 | zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
|
|---|
| 46 | zipfiletmp=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
|
|---|
| 47 | # check if zipped file already exists
|
|---|
| 48 | if [ -e $zipfile ]
|
|---|
| 49 | then
|
|---|
| 50 | continue
|
|---|
| 51 | fi
|
|---|
| 52 |
|
|---|
| 53 | # check if raw file was accessed in the last 30 minutes
|
|---|
| 54 | isnew=`find $file -amin -30`
|
|---|
| 55 | if [ "$isnew" != "" ]
|
|---|
| 56 | then
|
|---|
| 57 | echo $file" is not older than 30 min => continue" >> $logfile 2>&1
|
|---|
| 58 | continue
|
|---|
| 59 | fi
|
|---|
| 60 |
|
|---|
| 61 | # check if file is already finished
|
|---|
| 62 | # original file on daq (if data was taken on daq
|
|---|
| 63 | origfile=`echo $file | sed -e 's/loc_data/daq/'`
|
|---|
| 64 | if [ -e $origfile ]
|
|---|
| 65 | then
|
|---|
| 66 | # get time of last modification as seconds since Epoch for both files
|
|---|
| 67 | timeorig=`stat -c %Y $origfile`
|
|---|
| 68 | timecopy=`stat -c %Y $file`
|
|---|
| 69 | # compare times
|
|---|
| 70 | if ! [ $timeorig -eq $timecopy ]
|
|---|
| 71 | then
|
|---|
| 72 | # if times are not the same, the file is still open => no zip
|
|---|
| 73 | echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1
|
|---|
| 74 | continue
|
|---|
| 75 | fi
|
|---|
| 76 | else
|
|---|
| 77 | # if the origfile doesn't exist, the data was probably written not on daq but on data
|
|---|
| 78 | echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1
|
|---|
| 79 | fi
|
|---|
| 80 |
|
|---|
| 81 | echo `date`": zipping "$file" to "$zipfile" ..." >> $logfile 2>&1
|
|---|
| 82 | # zip file to stdout and pipe it to outputfile
|
|---|
| 83 | if pigz -1 -c -f $file > $zipfiletmp;
|
|---|
| 84 | then
|
|---|
| 85 | # if successful, move temporary to final zipfile
|
|---|
| 86 | mv -v $zipfiletmp $zipfile >> $logfile 2>&1
|
|---|
| 87 | else
|
|---|
| 88 | # if not successful, remove temporary zipfile
|
|---|
| 89 | rm -v $zipfiletmp >> $logfile 2>&1
|
|---|
| 90 | fi
|
|---|
| 91 | done
|
|---|
| 92 | done
|
|---|
| 93 | echo "finished zipping..." >> $logfile 2>&1
|
|---|
| 94 |
|
|---|
