Index: trunk/DataCheck/QuickLook/Step1.sh =================================================================== --- trunk/DataCheck/QuickLook/Step1.sh (revision 17902) +++ trunk/DataCheck/QuickLook/Step1.sh (revision 17903) @@ -1,4 +1,9 @@ #!/bin/bash # + +# remarks: +# rsync-server still used +# move fileerror check to main-loop? + source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0" @@ -7,5 +12,5 @@ if [ "$certaindate" != "" ] then - checkstring=`echo $certaindate | grep -E -o '20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]'` + checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'` if [ "$checkstring" = "" ] then @@ -28,7 +33,7 @@ makedir $infopath echo "" > $infopath/runrow.txt -# create path for sequence files -seqpath=$anapath/sequences/$datepath -makedir $seqpath +# create path for callisto output +calpath=$anapath/callisto/$datepath +makedir $calpath rawpathnewdaq=/newdaq/raw/$datepath rawpath=/loc_data/raw/$datepath @@ -81,7 +86,7 @@ if [ ${diskusage[3]} -lt $disklimitdaq ] then - #echo "WARN less than 700 left on /raid10 on node "$HOSTNAME" ("${diskusage[3]}")" - printprocesslog "DISK less than 700 left on /raid10 on node "$HOSTNAME" ("${diskusage[3]}")" - #df -h /raid10 + echo "WARN less than 700 left on /raid10 on node "$HOSTNAME + printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME + df -h /raid10 finish fi @@ -101,16 +106,25 @@ fi fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) -callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` ) +callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` ) # get number of dataruns from DB query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1" numdataruns=`sendquery` +query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6" +numlpruns=`sendquery` +query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300" +numpedruns=`sendquery` +query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5" +numdrstime=`sendquery` +numpedruns=0 +#numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l` +numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l` # create raw directory on daq, if not yet there makedir $rawpath -#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns -printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns - -while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ] # || [ $numdataruns -ne 0 ] # FIXME: Logik ueberdenken u ueberarb +echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated +printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated + +while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb do # only continue with script @@ -119,7 +133,7 @@ check_daq - numdataruns=0 - #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns - printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns + numcalibrated=0 + echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated + printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated rsync_aux_file $drivefilenewdaq $drivefile @@ -142,10 +156,8 @@ nondrs=`basename $file | sed -e 's/[.]drs//g'` nondrsfile=`find $rawpath -name $nondrs.*z` - #echo "nondrs: "$nondrsfile tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` else tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` fi - #echo "tstop -"$tstop"-" # when stop time is 0, file is not closed # when an error is returned the tstop is empty @@ -185,19 +197,36 @@ fi - if [ "`echo $localfile | grep -o drs`" != "drs" ] + # for .drs.fits files no further treatment needed + if [ "`echo $localfile | grep -o drs`" == "drs" ] then - runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` - runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1` - roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` + continue + fi + + # treat other files (.fits.fz) + runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` + if [ "$runtype" != "data" ] + then + # skip a non-data run when it has not 1000 evts + # as this means probably an fad-loss + # and these runs are repeated in that case numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` - printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi" numevts "$numevts - if [ "$runtype" == "drs-time-upshifted" ] + if [ $numevts -ne 1000 ] then - printprocesslog "INFO file "$file" has runtype drs-time-upshifted -> continue " + printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")" continue fi - #echo $runtype" "$runnum - if [ "$runtype" == "data" ] - then + fi + + # get run number + runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1` + + # what is needed to process the different runs? + # P: run#(P), run#(drs-file) + # C: run#(C), run#(drs-file), run#(drs-time) + # D: run#(D), run#(drs-file), run#(drs-time), ? + # what is drs-file? pedestal, roi300, has drs.fits + callistolog=$calpath"/"$date"_"$runnum"-calibration.log" + case $runtype in + data) # treat D-runs if [ "$fileerror" = "yes" ] then @@ -206,59 +235,13 @@ continue fi - seqfile=$seqpath/${date}_${runnum}.seq - printprocesslog "INFO write data-seq "$seqfile - echo "# written by automatic analysis in LP" >$seqfile - echo "" >> $seqfile - echo "Sequence: "`echo $date | cut -c3-8`$runnum >> $seqfile - echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile - echo "" >> $seqfile - echo "DrsSequence: "$drsseq >> $seqfile - echo "" >> $seqfile -# echo $runrow" CalRuns" -# echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g' - echo "CalRuns: "`echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'` >> $seqfile - echo "PedRuns: "`echo $runrow | grep -E -o '[0-9]{3}pedestal300' | sed -e 's/pedestal300//g'` >> $seqfile - echo "DatRuns: "$runnum >> $seqfile - echo "" >> $seqfile - echo "DrsFiles: "$drsfile >> $seqfile - echo "" >> $seqfile - echo "#DrsFile: "$drsfile >> $seqfile - echo "" >> $seqfile - -# tstopi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` -# tstopf=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'` -# tstop=${tstopi}${tstopf} -# coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop} 2>/dev/null | tail -1 2>&1` ) -# if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ] -# then -# printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile -# #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile -# continue -# fi -# if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ] -# then -# printprocesslog "WARN coordinates "${coordinates[@]} -# #echo "WARN coordinates "${coordinates[@]} -# continue -# fi -# printprocesslog "DEBUG coordinates "${coordinates[@]} -# query="SELECT fSourceKEY FROM scheduling.source WHERE " -# query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 " -# query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 " -# sourcekey=`sendquery` -# if [ "$sourcekey" == "" ] -# then -# printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]} -# fi - printprocesslog "INFO counting callisto logs and data files +1." + # some accounting + printprocesslog "DEBUG counting callisto logs and set data files +1." # get number of callisto logs runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l` - # count data runs - numdataruns=`echo " $numdataruns + 1 " | bc -l` - #echo "numdata +1" + # count runs to be calibrated + numcalibrated=`echo " $numcalibrated + 1 " | bc -l` + printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]} - #echo "cal: "$runcallistocount" numdat: "$numdataruns" numcallog: "${#callistofiles[@]} - printprocesslog "INFO running callistos: "$runcallistocount" #data-runs: "$numdataruns" #callisto-logs: "${#callistofiles[@]} # do not overload system in case of a lot of files to be processed # numruncallistos is set in setup.fact.lp.data @@ -266,8 +249,8 @@ then printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue" - #echo "INFO "$runcallistocount" RunCallisto.sh are running -> continue" continue fi - callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$date"_"$runnum"-calibration.log" + + # starting calibration if ! [ -e $callistolog ] then @@ -277,62 +260,82 @@ rsync_aux_file $tempfilenewdaq $tempfile rsync_aux_file $humfilenewdaq $humfile - #printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile - #echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile - #`dirname $0`/RunCallisto.sh $sourcekey $seqfile & - printprocesslog "INFO starting RunCallisto.sh for "$seqfile - #echo "INFO starting RunCallisto.sh for "$seqfile - `dirname $0`/RunCallisto.sh $seqfile & + if [ -e $drstime ] + then + calfile=$calpath"/"$date"_"$runnum"_C.root" + printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" outpath "$outpath" calfile "$calfile + echo "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" outpath "$outpath" calfile "$calfile + `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $drstime $calpath $calfile & + fi fi continue - else - # skip a non-data run when it has not 1000 evts - # as this means probably an fad-loss - if [ $numevts -ne 1000 ] - then - printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")" - continue - fi - fi - printprocesslog "DEBUG runrow "$runrow" (from variable) " - runrow=`cat $infopath/runrow.txt` - printprocesslog "DEBUG runrow "$runrow" (from file) " - runrow=$runrow$runnum$runtype$roi"_" - echo $runrow > $infopath/runrow.txt - if echo $runrow | grep -E '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_' >/dev/null - then - runrow2=`echo $runrow | grep -E -o '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'` - run1=`echo $runrow2 | cut -d_ -f1 | sed -e 's/drs-pedestal1024//g'` - run2=`echo $runrow2 | cut -d_ -f2 | sed -e 's/drs-gain1024//g'` - run3=`echo $runrow2 | cut -d_ -f3 | sed -e 's/drs-pedestal1024//g'` - run4=`echo $runrow2 | cut -d_ -f4 | sed -e 's/drs-pedestal1024//g'` - run5=`echo $runrow2 | cut -d_ -f5 | sed -e 's/drs-time1024//g'` - run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'` - run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'` - seqfile=$seqpath/${date}_${run1}.drs.seq - printprocesslog "INFO write drs-seq "$seqfile - echo "# written by automatic analysis in LP" > $seqfile - echo "" >> $seqfile - echo "Sequence: "`echo $date | cut -c3-8`$run1 >> $seqfile - echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile - echo "" >> $seqfile - echo "CalRuns: "$run2 >> $seqfile - echo "PedRuns: "$run6" "$run7 >> $seqfile - echo "DatRuns: "$run5 >> $seqfile - echo "DrsRuns: "$run1" "$run3" "$run4 >> $seqfile - echo "DrsFiles: "$run3" "$run6 >> $seqfile - echo "" >> $seqfile - echo "#DrsFile: "$run6 >> $seqfile - echo "" >> $seqfile - echo "" > $infopath/runrow.txt - drsseq=$run1 - drsfile=$run6 - fi - if echo $runrow | grep -E '[0-9]{3}pedestal300_[0-9]{3}light-pulser-ext300_' >/dev/null - then - echo "" > $infopath/runrow.txt - fi - fi + ;; + pedestal) # treat P-runs + roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` + if [ $roi -eq 300 ] + then + # check drs-file + drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'` + if [ -e $drsfile ] + then + # set name of drs-file + drscalib=$drsfile + continue + #else + # not needed for QLA + #if ! [ -e $callistolog ] + #then + # pedfile=$calpath"/"$date"_"$runnum"-pedestal.root" + # # count runs to be calibrated + # numcalibrated=`echo " $numcalibrated + 1 " | bc -l` + # printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile + # echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile + # `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile & + #fi + fi + fi + ;; + light-pulser-ext) # treat C-runs + # do lp-treatment -> not needed for QLA + #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root" + #if ! [ -e $callistolog ] + #then + # if [ -e $drstime ] + # then + # # count runs to be calibrated + # numcalibrated=`echo " $numcalibrated + 1 " | bc -l` + # printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile + # echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile + # `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile & + # fi + #fi + ;; + drs-time) # treat C-runs + # do drs-timing calibration + drstime=$calpath"/"$date"_"$runnum"-drstime.root" + # starting calibration + if ! [ -e $callistolog ] + then + # count runs to be calibrated + numcalibrated=`echo " $numcalibrated + 1 " | bc -l` + printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime + echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime + `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime & + fi + ;; + drs-pedestal) # get drs-pedestal + roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` + drs=`$factpath/fitsdump -h $localfile 2>/dev/null | grep DRSCALIB | grep -E -o " T " ` + if [ $roi -eq 1024 ] && [ "$drs" == " T " ] + then + drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'` + fi + ;; + *) # other runs + printprocesslog "INFO file "$file" has runtype "$runtype" -> continue " + continue + ;; + esac done - printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numdataruns" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile " + printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile " # get new file lists @@ -340,7 +343,7 @@ files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) - callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` ) - #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns - printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns + callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` ) + echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated + printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated # wait and get new file lists @@ -350,23 +353,13 @@ printprocesslog "INFO wait 60 seconds." sleep 60 - #echo "sleep 60..." printprocesslog "INFO get new file lists for "$datepath files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` ) - callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` ) + callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` ) fi - #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns - printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns - - # this should not happen, but may in rare cases if data are deleted in the wrong order - if [ ${#files[@]} -lt ${#fileslocal[@]} ] - then - printprocesslog "ERROR numfiles on newdaq("${#files[@]}") < #files on daq ("${#fileslocal[@]}") - maybe deleting data in the wrong order?" - echo "ERROR numfiles on newdaq("${#files[@]}") < #files on daq ("${#fileslocal[@]}") - maybe deleting data in the wrong order?" - finish - fi + echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated + printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated done -