Ignore:
Timestamp:
05/14/17 20:58:29 (8 years ago)
Author:
Daniela Dorner
Message:
updated to run on newdata and create a second copy
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Transfer/ZipRawData.sh

    r18328 r18853  
    1010date >> $logfile
    1111
    12 if ! ls /daq/raw >/dev/null 2>&1
     12if ! ls /scratch/raw >/dev/null 2>&1
    1313then
    14    printprocesslog "ERROR /daq/raw is not available."
     14   printprocesslog "ERROR /scratch/raw is not available."
    1515   finish
    1616fi
    17 # if daq unavailable
    18 #if ! ls /newdaq/raw >/dev/null 2>&1
    19 #then
    20 #   printprocesslog "ERROR /newdaq/raw is not available."
    21 #   finish
    22 #fi
    2317
    2418# get dates
    … …  
    3731#source $HEADAS/headas-init.sh
    3832
    39 # files younger than $delaytime are not processed
     33# files younger than $delaytime minutes are not processed
    4034delaytime=5
    4135
    … …  
    4438do
    4539   echo "" >> $logfile 2>&1
    46    rawdir=/daq/raw/$date
    47 #   rawdir=/newdaq/raw/$date #if daq unavailable
     40   rawdir=/scratch/raw/$date
    4841   echo `date`": processing files in "$rawdir >> $logfile 2>&1
    4942   printprocesslog "INFO processing files in "$rawdir
    … …  
    6861      continue
    6962   fi
    70 
    71    zipdir=/loc_data/zipraw/$date
    72    # create output directory for zip
     63   
     64   rawdir2=/data1/raw/$date
     65   zipdir=/data2/zipraw/$date
     66   # create output directories
     67   makedir $rawdir2 >> $logfile 2>&1
    7368   makedir $zipdir >> $logfile 2>&1
    7469
    … …  
    7873   for file in $fitsfiles
    7974   do
    80       # check if more than ~ 10 GB are left on /loc_data
    81       diskusage=( `df -P /loc_data | grep /loc_data ` )
     75      # check if more than ~ 10 GB are left on output directories
     76      diskusage=( `df -P /data1 | grep /data1 ` )
    8277      if [ ${diskusage[3]} -lt 10000000 ]
    8378      then
    84          echo "WARN less than 10 GB left on /loc_data on data ("${diskusage[3]}")" >> $logfile 2>&1
    85          printprocesslog "WARN less than 10 GB left on /loc_data on data ("${diskusage[3]}")"
     79         echo "WARN less than 10 GB left on /data1 on newdata ("${diskusage[3]}")" >> $logfile 2>&1
     80         printprocesslog "WARN less than 10 GB left on /data1 on newdata ("${diskusage[3]}")"
     81         finish
     82      fi
     83      diskusage=( `df -P /data2 | grep /data2 ` )
     84      if [ ${diskusage[3]} -lt 10000000 ]
     85      then
     86         echo "WARN less than 10 GB left on /data2 on newdata ("${diskusage[3]}")" >> $logfile 2>&1
     87         printprocesslog "WARN less than 10 GB left on /data2 on newdata ("${diskusage[3]}")"
    8688         finish
    8789      fi
    … …  
    9597         continue
    9698      fi
    97 
     99     
     100      # first rsync from /scratch to /data1
     101      file2=`echo $file | sed -e 's/scratch/data1/'`
     102      if ! rsync -au -T $rsynctempdir $file $file2
     103      then
     104         printprocesslog "ERROR something went wrong with rsync of "$file
     105         rm $file2
     106         continue
     107      fi
     108     
    98109      iszipped=`echo $file | grep -o fz`
    99110      if [ "$iszipped" == "fz" ]
    … …  
    103114
    104115         # filename for file on data
    105          zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' `
    106          zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
    107 #         # if daq unavailable
    108 #         zipfile=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' `
    109 #         zipfiletmp=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
     116         zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' `
     117         zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits.fz/fits.fz.tmp/'`
    110118         
    111119         # check if file on data already exists
    … …  
    144152
    145153         # filename for temporary and final zipfile
    146          zipfile=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
    147          zipfiletmp=`echo $file | sed -e 's/daq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
    148 #         # if daq unavailable
    149 #         zipfile=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
    150 #         zipfiletmp=`echo $file | sed -e 's/newdaq/loc_data/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
     154         zipfile=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
     155         zipfiletmp=`echo $file | sed -e 's/scratch/data2/g' -e 's/raw/zipraw/' -e 's/fits/fits.tmp.gz/'`
    151156         
    152157         # check if zipped file already exists
Note: See TracChangeset for help on using the changeset viewer.