#!/bin/bash # # remarks: # rsync-server still used # move fileerror check to main-loop? source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0" # get date (before 18h there is no new data to be processed) if [ "$certaindate" != "" ] then checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'` if [ "$checkstring" = "" ] then echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)" finish fi datepath=$certaindate else datepath=`date --date="-19HOUR" +%Y/%m/%d` fi date=`echo $datepath | sed -e 's/\///g'` printprocesslog "INFO processing "$datepath auxpathnewdaq=/newdaq/aux/$datepath # create aux directory on daq, if not yet there auxpath=/loc_data/aux/$datepath makedir $auxpath # create path for info files needed for analysis infopath=$anapath/info/$datepath makedir $infopath echo "" > $infopath/runrow.txt # create path for callisto output calpath=$anapath/callisto/$datepath makedir $calpath rawpathnewdaq=/newdaq/raw/$datepath rawpath=/loc_data/raw/$datepath # needed auxiliary files: # drive file with information about current source position drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits drivefilenewdaq=$auxpathnewdaq/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits # drive file with information about tracking position drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits drivefilenewdaq2=$auxpathnewdaq/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits # file with magic weather information mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits mweatherfilenewdaq=$auxpathnewdaq/${date}.MAGIC_WEATHER_DATA.fits # file with trigger rates ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits ratesfilenewdaq=$auxpathnewdaq/${date}.FTM_CONTROL_TRIGGER_RATES.fits # file with trigger rates tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits tempfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_TEMPERATURE.fits # file with trigger rates humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits humfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_HUMIDITY.fits function rsync_aux_file() { if ls $1 >/dev/null 2>&1 then printprocesslog "INFO rsync "$1 # rsync # from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/ # to daq (/daq = /loc_data on daq) rsyncservernewdaq=`echo $1 | sed -e 's/^\//172.16.100.100::/'` # old #if ! rsync -a -T $rsynctempdir $1 $2 # new (workaround for problems on daq) if ! rsync -a -T $rsynctempdir $rsyncservernewdaq $2 then printprocesslog "WARN rsync of "$1" failed." fi else printprocesslog "WARN "$1" missing." fi } function check_daq() { diskusage=( `df -P /raid10 | grep raid10 ` ) # check if more than 700 GB are left on /loc_data if [ ${diskusage[3]} -lt $disklimitdaq ] then echo "WARN less than 700 left on /raid10 on node "$HOSTNAME printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME df -h /raid10 finish fi } check_daq printprocesslog "INFO get lists of raw files on newdaq and daq" files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) # to treat links use: #files=( `find -L $rawpathnewdaq -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` ) if [ ${#files[@]} -eq 0 ] then printprocesslog "INFO no raw files available yet for "$datepath finish fi fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` ) # get number of dataruns from DB query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1" numdataruns=`sendquery` query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6" numlpruns=`sendquery` query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300" numpedruns=`sendquery` query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5" numdrstime=`sendquery` numpedruns=0 #numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l` numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l` # create raw directory on daq, if not yet there makedir $rawpath echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb do # only continue with script # when there is more than 10% space on daq source `dirname $0`/../Sourcefile.sh check_daq numcalibrated=0 echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated rsync_aux_file $drivefilenewdaq $drivefile # files on newdaq for file in ${files[@]} do printprocesslog "processing "$file #echo "processing "$file localfile=`echo $file | sed -e 's/newdaq/loc_data/'` source `dirname $0`/../Sourcefile.sh # check if file is already transferred if ! ls $localfile >/dev/null 2>&1 then # check if it is drs-file # get stop time from raw-file if [ "`echo $file | grep -o drs`" == "drs" ] then nondrs=`basename $file | sed -e 's/[.]drs//g'` nondrsfile=`find $rawpath -name $nondrs.*z` tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` else tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` fi # when stop time is 0, file is not closed # when an error is returned the tstop is empty if [ "$tstop" == "0" ] || [ "$tstop" == "" ] then printprocesslog "WARN "$file" not yet closed." # if a file is not closed and not touched for 30 minutes, # it is assumed corrupted and still transferred fileaccessed=`find $file -amin -30` if ! [ "$fileaccessed" == "" ] then printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue" continue else printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes" fileerror="yes" fi fi # rsync # from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/ # to daq (/daq = /loc_data on daq) # to access rsync server via the dedicated network between # daq and newdaq, use 172.16.100.100::newdaq filersyncserver=`echo $file | sed -e 's/^\//172.16.100.100::/'` # old ##if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile #if ! rsync -a -T $rsynctempdir --bwlimit=$bwlimit $file $localfile # new if ! rsync -a -W -T $rsynctempdir --bwlimit=$bwlimit $filersyncserver $localfile then printprocesslog "ERROR something went wrong with rsync of "$file rm $localfile continue fi printprocesslog "INFO "$file" rsynced successfully." fi # for .drs.fits files no further treatment needed if [ "`echo $localfile | grep -o drs`" == "drs" ] then continue fi # treat other files (.fits.fz) runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` if [ "$runtype" != "data" ] then # skip a non-data run when it has not 1000 evts # as this means probably an fad-loss # and these runs are repeated in that case numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` if [ $numevts -ne 1000 ] then printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")" continue fi fi # get run number runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1` # what is needed to process the different runs? # P: run#(P), run#(drs-file) # C: run#(C), run#(drs-file), run#(drs-time) # D: run#(D), run#(drs-file), run#(drs-time), ? # what is drs-file? pedestal, roi300, has drs.fits callistolog=$calpath"/"$date"_"$runnum"-calibration.log" case $runtype in data) # treat D-runs if [ "$fileerror" = "yes" ] then printprocesslog "INFO do not further process corrupted file "$localfile fileerror= continue fi # some accounting printprocesslog "DEBUG counting callisto logs and set data files +1." # get number of callisto logs runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l` # count runs to be calibrated numcalibrated=`echo " $numcalibrated + 1 " | bc -l` printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]} # do not overload system in case of a lot of files to be processed # numruncallistos is set in setup.fact.lp.data if [ $runcallistocount -ge $numruncallistos ] then printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue" continue fi # starting calibration if ! [ -e $callistolog ] then rsync_aux_file $drivefilenewdaq2 $drivefile2 rsync_aux_file $mweatherfilenewdaq $mweatherfile rsync_aux_file $ratesfilenewdaq $ratesfile rsync_aux_file $tempfilenewdaq $tempfile rsync_aux_file $humfilenewdaq $humfile if [ -e $drstime ] then calfile=$calpath"/"$date"_"$runnum"_C.root" printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" outpath "$outpath" calfile "$calfile echo "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" outpath "$outpath" calfile "$calfile `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $drstime $calpath $calfile & fi fi continue ;; pedestal) # treat P-runs roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` if [ $roi -eq 300 ] then # check drs-file drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'` if [ -e $drsfile ] then # set name of drs-file drscalib=$drsfile continue #else # not needed for QLA #if ! [ -e $callistolog ] #then # pedfile=$calpath"/"$date"_"$runnum"-pedestal.root" # # count runs to be calibrated # numcalibrated=`echo " $numcalibrated + 1 " | bc -l` # printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile # echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile # `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile & #fi fi fi ;; light-pulser-ext) # treat C-runs # do lp-treatment -> not needed for QLA #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root" #if ! [ -e $callistolog ] #then # if [ -e $drstime ] # then # # count runs to be calibrated # numcalibrated=`echo " $numcalibrated + 1 " | bc -l` # printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile # echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile # `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile & # fi #fi ;; drs-time) # treat C-runs # do drs-timing calibration drstime=$calpath"/"$date"_"$runnum"-drstime.root" # starting calibration if ! [ -e $callistolog ] then # count runs to be calibrated numcalibrated=`echo " $numcalibrated + 1 " | bc -l` printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime & fi ;; drs-pedestal) # get drs-pedestal roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` drs=`$factpath/fitsdump -h $localfile 2>/dev/null | grep DRSCALIB | grep -E -o " T " ` if [ $roi -eq 1024 ] && [ "$drs" == " T " ] then drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'` fi ;; *) # other runs printprocesslog "INFO file "$file" has runtype "$runtype" -> continue " continue ;; esac done printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile " # get new file lists printprocesslog "INFO get new file lists for "$datepath files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` ) echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated # wait and get new file lists update= if [ ${#fileslocal[@]} -eq ${#files[@]} ] then printprocesslog "INFO wait 60 seconds." sleep 60 printprocesslog "INFO get new file lists for "$datepath files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` ) fi echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated done