Changeset 18869


Ignore:
Timestamp:
05/21/17 10:32:05 (8 years ago)
Author:
Daniela Dorner
Message:
fixed bug and implemented option to use /data1 instead of /scratch
File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/DataCheck/QuickLook/Step1.sh

    r18855 r18869  
    11#!/bin/bash
    22#
     3
     4# to use script with /data1 instead of /scratch
     5#  (e.g. when /scratch is full)
     6# export data1=yes
     7# before executing the script
     8# preferably ZipRawData.sh has processed at this point all
     9#   files available already on /scratch
    310
    411# remarks:
     
    3643makedir $calpath >/dev/null
    3744rawpathnewdaq=/newdaq/raw/$datepath
    38 rawpath=/loc_data/raw/$datepath
     45if [ "$data1" = "yes" ]
     46then
     47   rawpath=/data1/raw/$datepath
     48   rsynctempdir=/data1/rsync_tmp
     49   printprocesslog "INFO using "$rawpath" for processing"
     50else
     51   rawpath=/scratch/raw/$datepath
     52   rsynctempdir=/scratch/rsync_tmp
     53fi
     54if ! [ -d $rsynctempdir ]
     55then
     56   mkdir $rsynctempdir
     57fi
     58
     59
    3960
    4061# needed auxiliary files:
     
    6687function check_disks()
    6788{
    68    # put here the check for /scratch (output of rsync) and /data1 (output of analysis)
     89   # at least 5% free disk on /data1
     90   diskusage=( `df -P /data1 | grep data1 ` )
     91   if [ ${diskusage[3]} -lt $disklimitnewdata2 ]
     92   then
     93      echo "WARN less than 5% left on /data1 on node "$HOSTNAME
     94      printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME
     95      df -h /data1
     96      finish
     97   fi
     98   
     99   # if /data1 is used for processing, /scratch doesn't need to be checked
     100   if [ "$data1" = "yes" ]
     101   then
     102      return
     103   fi
    69104   
    70105   # at least 10% free disk on /scratch
     
    77112      finish
    78113   fi
    79    # at least 5% free disk on /data1
    80    diskusage=( `df -P /data1 | grep data1 ` )
    81    if [ ${diskusage[3]} -lt $disklimitnewdata2 ]
    82    then
    83       echo "WARN less than 5% left on /data1 on node "$HOSTNAME
    84       printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME
    85       df -h /scratch
    86       finish
    87    fi
    88114}
    89115
     
    92118# getting lists of files
    93119printprocesslog "INFO get lists of raw files on newdaq and daq"
    94 files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
     120#files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
     121files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
    95122if [ ${#files[@]} -eq 0 ]
    96123then
     
    125152   #  when there is more than 10% space on daq
    126153   source `dirname $0`/../Sourcefile.sh
    127    check_disks
    128154   
    129155   numcalibrated=0
     
    137163   for file in ${files[@]}
    138164   do
     165      # check if still enough diskspace for transfer
     166      check_disks
     167      if [ "$certaindate" != "" ]
     168      then
     169         echo "processing "$file
     170      fi
    139171      printprocesslog "processing "$file
    140       echo "processing "$file
    141       #localfile=`echo $file | sed -e 's/newdaq/scratch/'`
    142       localfile=$file
     172      if [ "$data1" = "yes" ]
     173      then
     174         localfile=`echo $file | sed -e 's/newdaq/data1/'`
     175      else
     176         localfile=`echo $file | sed -e 's/newdaq/scratch/'`
     177      fi
     178      #localfile=$file
    143179
    144180      source `dirname $0`/../Sourcefile.sh
     
    176212
    177213         # rsync
    178          if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile
     214         #if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile
     215         if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
    179216         then
    180217            printprocesslog "ERROR something went wrong with rsync of "$file
     
    231268
    232269      # get run number
    233       runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
     270      runnum=`basename $localfile | cut -d_ -f2 | cut -d. -f1`
    234271     
    235272      # what is needed to process the different runs?
     
    351388   # get new file lists
    352389   printprocesslog "INFO get new file lists for "$datepath
    353    files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
     390   #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
     391   files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
    354392   fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
    355393   callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
     
    364402      sleep 60
    365403      printprocesslog "INFO get new file lists for "$datepath
    366       files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
     404      #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
     405      files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
    367406      fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
    368407      callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
Note: See TracChangeset for help on using the changeset viewer.