#!/bin/bash # # to use script with /data1 instead of /scratch # (e.g. when /scratch is full) # export data1=yes # before executing the script # preferably ZipRawData.sh has processed at this point all # files available already on /scratch # remarks: # move fileerror check to main-loop? source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0" # get date (before 18h there is no new data to be processed) if [ "$certaindate" != "" ] then checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'` if [ "$checkstring" = "" ] then echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)" finish fi datepath=$certaindate else # this has to be coordinated with the time when # ClearNewdata.sh is running datepath=`date --date="-18HOUR" +%Y/%m/%d` fi date=`echo $datepath | sed -e 's/\///g'` printprocesslog "INFO processing "$datepath auxpath=/loc_data/aux/$datepath makedir $auxpath >/dev/null # create path for info files needed for analysis infopath=$anapath/info/$datepath makedir $infopath >/dev/null echo "" > $infopath/runrow.txt # create path for callisto output calpath=$anapath/callisto/$datepath makedir $calpath >/dev/null rawpathnewdaq=/newdaq/raw/$datepath if [ "$data1" = "yes" ] then rawpath=/data1/raw/$datepath rsynctempdir=/data1/rsync_tmp printprocesslog "INFO using "$rawpath" for processing" else rawpath=/scratch/raw/$datepath rsynctempdir=/scratch/rsync_tmp fi if ! [ -d $rsynctempdir ] then mkdir $rsynctempdir fi # needed auxiliary files: # drive file with information about current source position drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits # drive file with information tracking position drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits # file with magic weather information mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits # file with trigger rates ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits # file with trigger rates tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits # file with trigger rates humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits function rsync_aux_file() { if check_file_avail $1 then printprocesslog "INFO rsync "$1 if ! rsync -a -T $rsynctempdir newdaq:$1 $1 then printprocesslog "WARN rsync of "$1" failed." fi fi } function check_disks() { # at least 5% free disk on /data1 diskusage=( `df -P /data1 | grep data1 ` ) if [ ${diskusage[3]} -lt $disklimitnewdata2 ] then echo "WARN less than 5% left on /data1 on node "$HOSTNAME printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME df -h /data1 finish fi # if /data1 is used for processing, /scratch doesn't need to be checked if [ "$data1" = "yes" ] then return fi # at least 10% free disk on /scratch diskusage=( `df -P /scratch | grep scratch ` ) if [ ${diskusage[3]} -lt $disklimitnewdata ] then echo "WARN less than 10% left on /scratch on node "$HOSTNAME printprocesslog "WARN less than 10% left on /scratch on node "$HOSTNAME df -h /scratch finish fi } check_disks # getting lists of files printprocesslog "INFO get lists of raw files on newdaq and daq" #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` ) if [ ${#files[@]} -eq 0 ] then printprocesslog "INFO no raw files available yet for "$datepath finish fi fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` ) # get number of dataruns from DB query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1" numdataruns=`sendquery` query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6" numlpruns=`sendquery` query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300" numpedruns=`sendquery` query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5" numdrstime=`sendquery` numpedruns=0 #numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l` numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l` # create raw directory on daq, if not yet there makedir $rawpath >/dev/null #echo "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb do # only continue with script # when there is more than 10% space on daq source `dirname $0`/../Sourcefile.sh numcalibrated=0 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated rsync_aux_file $drivefile rsync_aux_file $drivefile2 # files on newdaq for file in ${files[@]} do # check if still enough diskspace for transfer check_disks if [ "$certaindate" != "" ] then echo "processing "$file fi printprocesslog "processing "$file if [ "$data1" = "yes" ] then localfile=`echo $file | sed -e 's/newdaq/data1/'` else localfile=`echo $file | sed -e 's/newdaq/scratch/'` fi #localfile=$file source `dirname $0`/../Sourcefile.sh # check if file is already transferred if ! ls $localfile >/dev/null 2>&1 then # check if it is drs-file # get stop time from raw-file if [ "`echo $file | grep -o drs`" == "drs" ] then nondrs=`basename $file | sed -e 's/[.]drs//g'` nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null ` tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` else tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` fi # when stop time is 0, file is not closed # when an error is returned the tstop is empty if [ "$tstop" == "0" ] || [ "$tstop" == "" ] then printprocesslog "DEBUG "$file" not yet closed." # if a file is not closed and not touched for 30 minutes, # it is assumed corrupted and still transferred fileaccessed=`find $file -amin -30 2>/dev/null ` if ! [ "$fileaccessed" == "" ] then printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue" continue else printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes" fileerror="yes" fi fi # rsync #if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit $file $localfile then printprocesslog "ERROR something went wrong with rsync of "$file rm $localfile continue fi printprocesslog "INFO "$file" rsynced successfully." fi # for .drs.fits files no further treatment needed if [ "`echo $localfile | grep -o drs`" == "drs" ] then continue fi # # temporary check (needed to run on newdaq) # if [ "`echo $file | grep -o drs`" == "drs" ] # then # nondrs=`basename $file | sed -e 's/[.]drs//g'` # nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null ` # tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` # else # tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` # fi # # when stop time is 0, file is not closed # # when an error is returned the tstop is empty # if [ "$tstop" == "0" ] || [ "$tstop" == "" ] # then # printprocesslog "WARN "$file" not yet closed. -> continue (temporary check)" # continue # fi # # end temporary check # treat other files (.fits.fz) runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` if [ "$runtype" != "data" ] then # skip a non-data run when it has not 1000 evts # as this means probably an fad-loss # and these runs are repeated in that case numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` if [ "$numevts" == "" ] then printprocesslog "WARN could not get number of events from file "$file" -> continue " #echo "WARN could not get number of events from file "$file" -> continue " continue fi if [ $numevts -ne 1000 ] then printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")" continue fi fi # get run number runnum=`basename $localfile | cut -d_ -f2 | cut -d. -f1` # what is needed to process the different runs? # P: run#(P), run#(drs-file) # C: run#(C), run#(drs-file), run#(drs-time) # D: run#(D), run#(drs-file), run#(drs-time), ? # what is drs-file? pedestal, roi300, has drs.fits callistolog=$calpath"/"$date"_"$runnum"-calibration.log" case $runtype in data) # treat D-runs if [ "$fileerror" = "yes" ] then printprocesslog "INFO do not further process corrupted file "$localfile fileerror= continue fi # some accounting printprocesslog "DEBUG counting callisto logs and set data files +1." # get number of callisto logs runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l` # count runs to be calibrated numcalibrated=`echo " $numcalibrated + 1 " | bc -l` printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]} # do not overload system in case of a lot of files to be processed # numruncallistos is set in setup.fact.lp.data if [ $runcallistocount -ge $numruncallistos ] then printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue" continue fi # starting calibration if ! [ -e $callistolog ] then rsync_aux_file $drivefile rsync_aux_file $drivefile2 rsync_aux_file $mweatherfile rsync_aux_file $ratesfile rsync_aux_file $tempfile rsync_aux_file $humfile calfile=$calpath"/"$date"_"$runnum"_C.root" printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" outpath "$outpath" calfile "$calfile `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $calpath $calfile & fi continue ;; pedestal) # treat P-runs roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` if [ $roi -eq 300 ] then # check drs-file drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'` if [ -e $drsfile ] then # set name of drs-file drscalib=$drsfile continue #else # not needed for QLA #numcalibrated=`echo " $numcalibrated + 1 " | bc -l` #if ! [ -e $callistolog ] #then # pedfile=$calpath"/"$date"_"$runnum"-pedestal.root" # # count runs to be calibrated # printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile # echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile # `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile & #fi fi fi ;; light-pulser-ext) # treat C-runs # do lp-treatment -> not needed for QLA #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root" #numcalibrated=`echo " $numcalibrated + 1 " | bc -l` #if ! [ -e $callistolog ] #then # if [ -e $drstime ] # then # # count runs to be calibrated # printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile # echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile # `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile & # fi #fi ;; drs-time) # treat drs-time runs # do drs-timing calibration drstime=$calpath"/"$date"_"$runnum"-drstime.root" # starting calibration numcalibrated=`echo " $numcalibrated + 1 " | bc -l` if ! [ -e $callistolog ] then # count runs to be calibrated printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime #echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime & fi ;; drs-pedestal) # get drs-pedestal roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` drs=`$factpath/fitsdump -h $localfile 2>/dev/null | grep DRSCALIB | grep -E -o " T " ` if [ $roi -eq 1024 ] && [ "$drs" == " T " ] then drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'` fi ;; *) # other runs printprocesslog "INFO file "$file" has runtype "$runtype" -> continue " continue ;; esac done printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile " # get new file lists printprocesslog "INFO get new file lists for "$datepath #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` ) fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` ) #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated # wait and get new file lists update= if [ ${#fileslocal[@]} -eq ${#files[@]} ] then printprocesslog "INFO wait 60 seconds." sleep 60 printprocesslog "INFO get new file lists for "$datepath #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` ) files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` ) fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` ) fi #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated sleep 30 printprocesslog "INFO sleep 30" done finish