Index: trunk/DataCheck/Processing/CheckRawData.sh =================================================================== --- trunk/DataCheck/Processing/CheckRawData.sh (revision 18082) +++ trunk/DataCheck/Processing/CheckRawData.sh (revision 18083) @@ -1,22 +1,10 @@ #!/bin/bash -# this script has been written to run on La Palma on the machine data -# i.e. paths are only working on this machine -# the script starts from the zipped files -# this causes a delay until files are in the database -# because they have to be rsynced and zipped first (see RsyncRawData.sh, ZipRawData.sh) -# -# starting from 2012/06/05 the keywords CHECKSUM and DATASUM are -# available in the fits header of the rawfile -# starting from this time the md5sums are not calculated anymore (not true, only from 2012/12/17) -# +# completely new version of the script +# running on newdaq to fill the information as fast as possible +# ftools is not used in this script anymore # todo # tempfile for fitsdump -h output - -## options: -#skipmd5sum="no" # fill md5 sums in any case -#skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default -#skipmd5sum="yes" # do not fill md5 sums in any case #new default since 2012/06/05 when the checksum is available in heaser # option whether to fill all row or only those where information is missing @@ -28,12 +16,6 @@ fi - -root=/opt/root_svn/bin/thisroot.sh - source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0 with options doupdate="$doupdate #" and skipmd5sum="$skipmd5sum - -# setup to use ftools -source $HEADAS/headas-init.sh # check if software is available @@ -45,17 +27,7 @@ # check if paths are available -if ! ls /daq/raw >/dev/null 2>&1 +if ! ls $rawdata >/dev/null 2>&1 then - printprocesslog "ERROR /daq/raw is not available." - finish -fi -if ! ls /newdaq/raw >/dev/null 2>&1 -then - printprocesslog "ERROR /newdaq/raw is not available." - finish -fi -if ! ls /loc_data/zipraw >/dev/null 2>&1 -then - printprocesslog "ERROR /loc_data/zipraw is not available." + printprocesslog "ERROR "$rawdata" is not available." finish fi @@ -81,72 +53,37 @@ for date in ${dates[@]} do - date2=`echo $date | sed -e 's/\///g'` - night= - query="SELECT fNight FROM AuxFilesAvailISDCStatus WHERE fNight="$date2 - night=`sendquery` - if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1 - then - printprocesslog "INFO insert "$date2" to AuxFilesAvailISDCStatus" - query="INSERT AuxFilesAvailISDCStatus SET fNight="$date2", fPriority="$date2 - sendquery >/dev/null 2>&1 - fi - night= - query="SELECT fNight FROM DriveFileAvailISDCStatus WHERE fNight="$date2 - night=`sendquery` - if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1 - then - printprocesslog "INFO insert "$date2" to DriveFileAvailISDCStatus" - query="INSERT DriveFileAvailISDCStatus SET fNight="$date2", fPriority="$date2 - sendquery >/dev/null 2>&1 - fi - night= - query="SELECT fNight FROM RatesFileAvailISDCStatus WHERE fNight="$date2 - night=`sendquery` - if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1 - then - printprocesslog "INFO insert "$date2" to RatesFileAvailISDCStatus" - query="INSERT RatesFileAvailISDCStatus SET fNight="$date2", fPriority="$date2 - sendquery >/dev/null 2>&1 - fi - night= - query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$date2 - night=`sendquery` - if [ "$night" == "" ] && ls /loc_data/zipraw/$date >/dev/null 2>&1 - then - printprocesslog "INFO insert "$date2" to AuxDataInsertStatus" - query="INSERT AuxDataInsertStatus SET fNight="$date2", fPriority="$date2 - sendquery >/dev/null 2>&1 - fi - ziprawdir=/loc_data/zipraw/$date - # check if data are available from that night - if ! [ -d $ziprawdir ] - then - printprocesslog "INFO "$ziprawdir" does not exist." - continue - else - printprocesslog "INFO processing "$ziprawdir"..." - fi - - # find all fits.gz files starting with the oldest file - printprocesslog "INFO finding files to be checked in $ziprawdir..." - fitsgzfiles=`find $ziprawdir -type f -name '*.fits.gz'| sort ` - # get runnumber from date runnumber=`echo $date | sed -e 's/\///g'` + night= + query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$runnumber + night=`sendquery` + if [ "$night" == "" ] && ls $rawdata/$date >/dev/null 2>&1 + then + printprocesslog "INFO insert "$runnumber" to AuxDataInsertStatus" + query="INSERT AuxDataInsertStatus SET fNight="$runnumber", fPriority="$runnumber + sendquery >/dev/null 2>&1 + fi + rawdir=$rawdata/$date + # check if data are available from that night + if ! [ -d $rawdir ] + then + printprocesslog "INFO "$rawdir" does not exist." + continue + else + printprocesslog "INFO processing "$rawdir"..." + fi + + # find all fits.gz files starting with the oldest file + printprocesslog "INFO finding files to be checked in $rawdir..." + #fitsfiles=`ls $rawdir/*.fits 2>/dev/null | sort ` + fitsfiles=`find $rawdir -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` + # loop to check files - for file in $fitsgzfiles + for file in $fitsfiles do - filecorrupt="no" printprocesslog "INFO checking file "$file - #echo "INFO checking file "$file" at "`date` - - # raw and original file - # file: /loc_data/zipraw - # rawfile: /daq/raw - # origfile: /newdaq/raw - rawfile=`echo $file | sed -e 's/loc_data/daq/' -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'` - rawfile2=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'` - origfile=`echo $rawfile | sed -e 's/daq/newdaq/'` + #echo "INFO checking file "$file + filenotclosed="no" # check if it is drs file @@ -158,35 +95,5 @@ fi -# not needed anymore as files are only rsynced from newdaq to daq when they are closed -# # check if file is already finished -# # original file on daq (if data was taken on daq -# if [ -e $origfile ] -# then -# # check if raw file was changed in the last 30 minutes -# isnew=`find $origfile -cmin -30` -# if [ "$isnew" != "" ] -# then -# printprocesslog "WARN "$origfile" is not older than 30 min. -> continue" -# continue -# fi -# -# # get time of last modification as seconds since Epoch for both files -# timeorig=`stat -c %Y $origfile` -# timecopy=`stat -c %Y $rawfile` -# # compare times -# if ! [ $timeorig -eq $timecopy ] -# then -# # if times are not the same, the file is still open => no check -# printprocesslog "INFO file "$rawfile" not yet closed. -> continue" -# continue -# fi -# else -# # if the origfile doesn't exist, the data was probably written not on daq but on data -# printprocesslog "INFO file "$rawfile" was probably taken on data and not daq." -# fi - # get run and file number form filename - runnumbererror="no" - numbererror="no" numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'` runnumberfromname=`echo $numberfromname | cut -d_ -f1` @@ -194,5 +101,4 @@ if [ "$runnumber" != "$runnumberfromname" ] then - runnumbererror="yes" printprocesslog "ERROR for file "$file": runnumber from date ("$runnumber") and filename ("$runnumberfromname") don't agree." fi @@ -214,48 +120,37 @@ runtype= - # check if fits file is corrupted - numfitserrors=0 - #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` - #numfitserrors=`fverify $rawfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` -# ftools missing on daq -# numfitserrors=`ssh daq "export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/ ; source $HEADAS/headas-init.sh ; fverify $rawfile2 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'"` -# echo $rawfile2" ------ "$numfitserrors - #if [ "$checkfitsfile" == "" ] - if [ $numfitserrors -gt 0 ] - then - printprocesslog "WARN "$rawfile2" has "$numfitserrors" fitserror(s). " - #fitsdumperrors=`$factpath/fitsdump -h -t Events $file 2>&1 | grep corrupted` - fitsdumperrors=`$factpath/fitsdump -h $file 2>&1 | grep corrupted` - if [ "$fitsdumperrors" != "" ] - then - filecorrupt="yes" - fi - #numfitserrors=1 - fi - # check if file was closed properly if [ "`echo $file | grep -o drs`" == "drs" ] then - nondrsfile=`echo $file | sed -e 's/[.]drs//g'` - tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"` + nondrsfile=`echo $file | sed -e 's/[.]drs//g'`".fz" # fixme: maybe more flexible + tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` + checkstatus=$? else - tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"` - fi - if [ "$tstop" == "0" ] - then - echo $file" not yet closed." - echo "WARN: $file has empty TSTOP" - filecorrupt="yes" + tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'` + checkstatus=$? + fi + #if [ $checkstatus -gt 0 ] + #then + # echo "WARN: getting tstop for file "$file" failed: tstop: -"$tstop"-"`date` + #fi + if [ "$tstop" == "0" ] || [ "$tstop" == "" ] + then + fileaccessed=`find $file -amin -30` + if ! [ "$fileaccessed" == "" ] + then + printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue" + continue + else + printprocesslog "WARN "$file" has empty TSTOP but was not touched for 30 minutes" + filenotclosed="yes" + fi fi numdrsfiles= step= - #if [ $numfitserrors -eq 0 ] - if [ "$filecorrupt" == "no" ] + if [ "$filenotclosed" == "no" ] then # get run and file number from file - #runnumberfromfile=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'` runnumberfromfile=`$factpath/fitsdump -h $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'` - #filenumberfromfileorig=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'` filenumberfromfileorig=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'` if [ "$runnumberfromfile" = "" ] || [ "$filenumberfromfileorig" = "" ] @@ -267,10 +162,8 @@ if [ "$numberfromfile" != "$numberfromname" ] then - numbererror="yes" printprocesslog "ERROR for file "$file": number from filename ("$numberfromname") and file ("$numberfromfile") don't agree." fi # get checksums from header - #checksum=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'` checksum=`$factpath/fitsdump -h $file 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'` if [ "$checksum" == "" ] @@ -278,5 +171,4 @@ printprocesslog "WARN checksum for file "$file" is empty." fi - #datasum=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'` datasum=`$factpath/fitsdump -h $file 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'` if [ "$datasum" == "" ] @@ -289,13 +181,10 @@ # in the very beginning only drs-files were existing # in the beginning the keywords DRSCALIB and STEP were not existing - drsfile=`echo $file | sed -e 's/fits/drs.fits/'` + drsfile=`echo $file | sed -e 's/fits.fz/drs.fits/'` numdrsfiles=`ls $drsfile 2>/dev/null | wc -l` - #drscalib=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DRSCALIB | grep -E -o "['][TF][']" | sed -e "s/'//g"` drscalib=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"` if [ "$drscalib" == "T" ] then - #step=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DRSSTEP | grep -E -o "['][012][']" | sed -e "s/'//g"` step=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"` - #stepfromdrs=`$factpath/fitsdump -h -t Events $drsfile 2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"` stepfromdrs=`$factpath/fitsdump -h $drsfile 2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"` if [ "$stepfromdrs" != "$step" ] @@ -322,33 +211,18 @@ fi fi - #echo $numdrsfiles" "$stepfromdrs" "$step" "$drscalib" ("$file")" # get other variables from header - #runtype=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` runtype=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"` - #echo "runtype for file "$file": "$runtype - #roi=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` roi=`$factpath/fitsdump -h $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` - #roitm=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'` roitm=`$factpath/fitsdump -h $file 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'` - #numevents=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` numevents=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` - #numphys=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'` numphys=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'` - #numext1=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` numext1=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` - #numext2=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` numext2=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` - #numelp=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'` numelp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'` - #numilp=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'` numilp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'` - #numoth=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'` numoth=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'` - #numped=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'` numped=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'` - #numtime=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'` numtime=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'` - #compiled=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"` compiled=`$factpath/fitsdump -h $file 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"` if ! [ "$compiled" == "" ] @@ -358,12 +232,9 @@ compiletime= fi - #revnum=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"` revnum=`$factpath/fitsdump -h $file 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"` # in newest data start time is in DATE-OBS # in older data start time is in TSTART # in the beginning TSTART was empty - #runstart=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - #runstart2=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` runstart2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` if [ "$runstart" == "" ] @@ -371,6 +242,7 @@ if [ "$runstart2" == "" ] then - #runstart=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + # for very old data, but gives problems for newer data + #runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstart="0000-00-00 00:00:00" else runstart=$runstart2 @@ -380,7 +252,5 @@ # in older data start time is in TSTOP # in the beginning TSTOP was empty - #runstop=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` runstop=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - #runstop2=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` runstop2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` if [ "$runstop" == "" ] @@ -388,5 +258,7 @@ if [ "$runstop2" == "" ] then - runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` + # for very old data, but gives problems for newer data + #runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` + runstop="0000-00-00 00:00:00" else runstop=$runstop2 @@ -427,5 +299,4 @@ query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'" printprocesslog "DEBUG get run type from DB. QUERY:"$query - #result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` ) result2=( `sendquery` ) if [ ${#result2} -eq 0 ] @@ -434,34 +305,4 @@ continue fi - -# # check if entry has already checksums -# query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'" -# query=$query" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)" -# printprocesslog "DEBUG check if md5sums are alreay in DB. QUERY:"$query -# #result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"` -# result5=`sendquery` -# -# # get md5sums of raw and zip file -# # to safe time for tests and update this can be skipped ($skipmd5sum) -# md5sum= -# md5sumzip= -# if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ] -# then -# #echo "skip: "$skipmd5sum >> $logfile 2>&1 -# #echo "res5: -"$result5"-" >> $logfile 2>&1 -# #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ] -# if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ] -# then -# #zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` -# # only do the md5sum if the zipfile is already available -# if ls $file >/dev/null 2>&1 -# then -# printprocesslog "INFO calculation md5sum for rawfile "$rawfile -# md5sum=`md5sum $rawfile | cut -d' ' -f1` -# printprocesslog "INFO calculation md5sum for zipfile "$file -# md5sumzip=`md5sum $file | cut -d' ' -f1` -# fi -# fi -# fi # insert or update depending on whether run exists @@ -477,6 +318,5 @@ fi query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]} - #if [ $numfitserrors -eq 0 ] - if [ "$filecorrupt" == "no" ] + if [ "$filenotclosed" == "no" ] then query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'" @@ -534,8 +374,4 @@ query=$query", fDataSum='"$datasum"'" fi -# if [ "$md5sum" != "" ] -# then -# query=$query", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'" -# fi if [ "$numdrsfiles" != "" ] then @@ -554,5 +390,5 @@ query=$query", fRevisionNumber='"$revnum"'" fi - query=$query", fFitsFileErrors="$numfitserrors + #query=$query", fFitsFileErrors="$numfitserrors query=$query" "$querystop # send query to DB Index: trunk/DataCheck/Processing/FillAuxData.sh =================================================================== --- trunk/DataCheck/Processing/FillAuxData.sh (revision 18082) +++ trunk/DataCheck/Processing/FillAuxData.sh (revision 18083) @@ -1,7 +1,47 @@ #!/bin/bash -# missing: -# vferify of files -# only treat file if it is there +# new version of the script to run on newdaq + +# option whether to fill all row or only those where information is missing +# $doupdate might be given as environment variable +if [ "$doupdate" = "" ] +then + doupdate="yes" # update all entries (needed when new fields have been added) + doupdate="no" # fill only entries which are not yet existing (default) +fi + +source `dirname $0`/../Sourcefile.sh +printprocesslog "INFO starting $0 with option doupdate="$doupdate + +logfile=$runlogpath"/FillAuxData-"$datetime".log" +date >> $logfile + +# check if software is available +if ! ls $factpath/fitsdump >/dev/null 2>&1 +then + printprocesslog "ERROR "$factpath"/fitsdump is not available." + finish +fi + +# get dates +if [ "$certaindate" != "" ] +then + checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'` + if [ "$checkstring" = "" ] + then + echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)" + finish + fi + getdates $certaindate +else + # get all night + #getdates "all" + # get last 3 nights if hour between 7 and 19h, else only current night + getdates 3 7 19 +fi + + +printprocesslog "INFO processing the following night(s): "${dates[@]} +echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1 # mjd of 1970-01-01 @@ -20,99 +60,33 @@ # trigger rate has as first value -1, but with using the median it should be fine -# option whether to fill all row or only those where information is missing -# $doupdate might be given as environment variable -if [ "$doupdate" = "" ] -then - doupdate="yes" # update all entries (needed when new fields have been added) - doupdate="no" # fill only entries which are not yet existing (default) -fi - -source `dirname $0`/../Sourcefile.sh -printprocesslog "INFO starting $0 with option doupdate="$doupdate - -logfile=$runlogpath"/FillAuxData-"$datetime".log" -date >> $logfile - -function getfitsstatistics() + +function evaluatestatistics() { - # $1 filename - # $2 colname - # $3 tstart - # $4 tstop - good= + # $1 variable name + # $@ statistics + if [ "$2" = "" ] + then + printprocesslog "WARN couldn't get statistics from file $1 for run "$date" "$file + continue + fi min= mean= - median= + med= max= - tmpfile=`dirname $0`/`basename $1`.tmp - echo "ftcopy $1'[col Time]' - | ftstat - | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'" >> $logfile - timefromfile=`ftcopy $1'[col Time]' - 2>>$logfile | ftstat - 2>>$logfile | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'` - echo "here-"$timefromfile"-" >> $logfile - if [ "$timefromfile" == "" ] - then - printprocesslog "WARN couldn't get time from file "$1 - echo "WARN couldn't get time from file "$1 - continue - fi - if [ $timefromfile -gt 30000 ] - then - echo "ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile - #ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - - ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile - else - echo "ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile - #ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - - ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile - fi - good=`cat $tmpfile | grep 'good' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` - min=`cat $tmpfile | grep 'min' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` - mean=`cat $tmpfile | grep 'mean' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` - median=`cat $tmpfile | grep 'median' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` - max=`cat $tmpfile | grep 'max' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` - sigma=`cat $tmpfile | grep 'sigma' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` - if [ "$good" = "" ] - then - printprocesslog "WARN couldn't get statistics from file $1 for run $date $file" - rm $tmpfile - continue - #finish - fi - #echo "good: "$good - #echo "min: "$min - #echo "max: "$max - #echo "mean: "$max - #echo "median: "$max - rm $tmpfile + rms= + evaluation=`echo $@ | grep -E -o '\['${1}':0[:]?[0-9]*\]\ Min:\ [-]?[0-9]+[.]?[0-9]*\ Max:\ [-]?[0-9]+[.]?[0-9]*\ Med:\ [-]?[0-9]+[.]?[0-9]*\ Avg:\ [-]?[0-9]+[.]?[0-9]*\ Rms:\ [-]?[0-9]+[.]?[0-9]*[e]?[-]?[0-9]*'` + if [ "$evaluation" = "" ] + then + printprocesslog "WARN empty evaluation of statistic ("$@") for run "$date" "$file + #echo " ---> "$@ + fi + #echo "ev: "$evaluation + min=`echo $evaluation | grep -E -o 'Min:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Min:\ //'` + max=`echo $evaluation | grep -E -o 'Max:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Max:\ //'` + med=`echo $evaluation | grep -E -o 'Med:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Med:\ //'` + mean=`echo $evaluation | grep -E -o 'Avg:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Avg:\ //'` + rms=`echo $evaluation | grep -E -o 'Rms:\ [-]?[0-9]+[.]?[0-9]*[e]?[-]?[0-9]+' | sed -e 's/Rms:\ //'` + #echo "eval: "$min" "$max" "$med" "$mean" "$rms } - -# setup to use ftools -source $HEADAS/headas-init.sh - -# check if software is available -if ! ls $factpath/fitsdump >/dev/null 2>&1 -then - printprocesslog "ERROR "$factpath"/fitsdump is not available." - finish -fi - -# get dates -if [ "$certaindate" != "" ] -then - checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'` - if [ "$checkstring" = "" ] - then - echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)" - finish - fi - getdates $certaindate -else - # get all night - #getdates "all" - # get last 3 nights if hour between 7 and 19h, else only current night - getdates 3 7 19 -fi - -printprocesslog "INFO processing the following night(s): "${dates[@]} -echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1 # do filling of aux data @@ -120,22 +94,6 @@ do auxdir=$auxdata/$date + rawdir=$rawdata/$date runnumber=`echo $date | sed -e 's/\///g'` - - # get file numbers from DB - # but only for not-corrupted files - query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 " - # only runs which are not yet filled - if [ "$doupdate" = "no" ] - then - query=$query" AND ISNULL(fRightAscension) " - fi - printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query - filenumbers=( `sendquery $query` ) - # proceed only if there are files available - if [ ${#filenumbers} -eq 0 ] - then - printprocesslog "INFO No files found in the DB for night "$date - continue - fi # check if aux files are available from that night @@ -146,4 +104,31 @@ else printprocesslog "INFO processing files in "$auxdir + #echo "INFO processing files in "$auxdir + fi + + # check if raw files are available from that night + # only needed to get start/stop time + # might be removed once the start/stop time comes from the DB + if ! [ -d $rawdir ] + then + printprocesslog "INFO no data available in "$rawdir" -> continue" + continue + fi + + # get file numbers from DB + # but only for not-corrupted files + # as aux files are written only once a minute, select only files which are older than 1.5 minutes + query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 AND NOT ISNULL(fRunStop) AND fRunStop < SUBTIME(UTC_TIMESTAMP(), \"00:01:30\")" + # only runs which are not yet filled + if [ "$doupdate" = "no" ] + then + query=$query" AND ISNULL(fRightAscension) " + fi + printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query + filenumbers=( `sendquery $query` ) + if [ ${#filenumbers} -eq 0 ] + then + printprocesslog "INFO No files found in the DB for night "$date + continue fi @@ -153,10 +138,4 @@ then printprocesslog "WARN "$trackingfile" not found." - else - tracknumerrors=`fverify $trackingfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` - if [ $tracknumerrors -gt 0 ] - then - printprocesslog "WARN for $trackingfile fverify returned "$tracknumerrors" error(s)." - fi fi @@ -166,9 +145,4 @@ printprocesslog "WARN "$sourceposfile" not found." else - sourceposnumerrors=`fverify $sourceposfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` - if [ $sourceposnumerrors -gt 0 ] - then - printprocesslog "WARN for $sourceposfile fverify returned "$sourceposnumerrors" error(s)." - fi sourceposfiletstarti=`$factpath/fitsdump -h $sourceposfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` sourceposfiletstartf=`$factpath/fitsdump -h $sourceposfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` @@ -179,5 +153,4 @@ sourceposfiletstart=`echo " $sourceposfiletstarti + $sourceposfiletstartf " | bc -l` fi - #echo "tstart: "$sourceposfiletstart fi @@ -186,10 +159,4 @@ then printprocesslog "WARN "$triggerratefile" not found." - else - trignumerrors=`fverify $triggerratefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` - if [ $trignumerrors -gt 0 ] - then - printprocesslog "WARN for $triggerratefile fverify returned "$trignumerrors" error(s)." - fi fi @@ -198,10 +165,4 @@ then printprocesslog "WARN "$thresholdfile" not found." - else - treshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` - if [ $treshnumerrors -gt 0 ] - then - printprocesslog "WARN for $thresholdfile fverify returned "$treshnumerrors" error(s)." - fi fi @@ -210,10 +171,4 @@ then printprocesslog "WARN "$biasvoltagefile" not found." - else - biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` - if [ $biasnumerrors -gt 0 ] - then - printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)." - fi fi @@ -223,6 +178,8 @@ printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum` echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1 + #echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` + # get information from rawfile - rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz + rawfile=`ls $rawdir/$runnumber"_"\`printf %03d $filenum\`.fits*` if ! [ -e $rawfile ] then @@ -230,13 +187,7 @@ continue fi - #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` - #if [ "$checkfitsfile" == "" ] - #then - # numfitserrors=1 - # printprocesslog "WARN: "$rawfile" probably corrupted." - # continue - #fi - runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` - mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + + #runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` + #mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` @@ -277,5 +228,5 @@ # get information from source_pos file - if [ -e $sourceposfile ] && [ $sourceposnumerrors -eq 0 ] + if [ -e $sourceposfile ] then sourcename=`$factpath/fitsdump ${sourceposfile} -c Time -c Name --filter='[1]<'${tstop}' && [1]>'${sourceposfiletstart} 2>/dev/null | tail -1 2>&1 | grep -o -E "['][a-zA-Z0-9\ \.\+\-]+[']" | sed -e "s/'//g"` @@ -295,4 +246,5 @@ # build query to update runinfo in DB query="UPDATE RunInfo SET " + # fill source key only if available if ! [ "$sourcekey" = "" ] @@ -304,33 +256,36 @@ # get information from tracking - if [ -e $trackingfile ] && [ $tracknumerrors -eq 0 ] - then + if [ -e $trackingfile ] + then + # get statistics + trackingstats=`$factpath/fitsdump $trackingfile -s -c Time -c Ra -c Dec -c Zd -c Az --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null` # RA - getfitsstatistics $trackingfile "Ra" $tstart $tstop - if [ "$min" == "$max" ] && [ $good -gt 0 ] - then - query=$query" fRightAscension="$mean - else - query=$query" fRightAscension=NULL" - if [ $good -gt 0 ] - then + evaluatestatistics "Ra" $trackingstats + #echo $min"_"$max"_"$ + if [ "$evaluation" != "" ] + then + if [ "$min" == "$max" ] + then + query=$query" fRightAscension="$mean + else + query=$query" fRightAscension=NULL" printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")." fi - fi - # Declination - getfitsstatistics $trackingfile "Dec" $tstart $tstop - if [ "$decmin" == "$decmax" ] && [ $good -gt 0 ] - then - query=$query", fDeclination="$mean - else - query=$query", fDeclination=NULL" - if [ $good -gt 0 ] - then + # Declination + evaluatestatistics "Dec" $trackingstats + if [ "$decmin" == "$decmax" ] + then + query=$query", fDeclination="$mean + else + query=$query", fDeclination=NULL" printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")." fi + else + query=$query" fRightAscension=NULL" + query=$query", fDeclination=NULL" fi # Zd - getfitsstatistics $trackingfile "Zd" $tstart $tstop - if [ $good -gt 0 ] + evaluatestatistics "Zd" $trackingstats + if [ "$evaluation" != "" ] then query=$query", fZenithDistanceMin="$min @@ -343,6 +298,6 @@ fi # Az - getfitsstatistics $trackingfile "Az" $tstart $tstop - if [ $good -gt 0 ] + evaluatestatistics "Az" $trackingstats + if [ "$evaluation" != "" ] then query=$query", fAzimuthMin="$min @@ -366,23 +321,12 @@ # get information from trigger - if [ -e $triggerratefile ] && [ $trignumerrors -eq 0 ] - then - #echo " $triggerratefile TriggerRate $tstart $tstop" - getfitsstatistics $triggerratefile "TriggerRate" $tstart $tstop -# if [ "$mjdreftrig" == "" ] -# then -# ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` -# ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` -# ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` -# ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` -# else -# ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` -# ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` -# ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` -# ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` -# fi - if [ $good -gt 0 ] - then - query=$query", fTriggerRateMedian="$median + if [ -e $triggerratefile ] + then + # get statistics + triggerstats=`$factpath/fitsdump $triggerratefile -s -c Time -c TriggerRate --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null` + evaluatestatistics "TriggerRate" $triggerstats + if [ "$evaluation" != "" ] + then + query=$query", fTriggerRateMedian="$med else query=$query", fTriggerRateMedian=NULL" @@ -393,14 +337,18 @@ # get information from thresholds - if [ -e $thresholdfile ] && [ $treshnumerrors -eq 0 ] - then - getfitsstatistics $thresholdfile "PatchThresh" $tstart $tstop - if [ $good -eq 0 ] - then - getfitsstatistics $thresholdfile "PatchThresh" $tstart2 $tstop - fi - if [ $good -gt 0 ] - then - query=$query", fThresholdMedian="$median + if [ -e $thresholdfile ] + then + # get statistics + thresholdstats=`$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null` + evaluatestatistics "PatchThresh" $thresholdstats + if [ "$evaluation" = "" ] + then + thresholdstats=`$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null` + #echo "$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null" + evaluatestatistics "PatchThresh" $thresholdstats + fi + if [ "$evaluation" != "" ] + then + query=$query", fThresholdMedian="$med else query=$query", fThresholdMedian=NULL" @@ -411,20 +359,27 @@ # get information from bias: U - if [ -e $biasvoltagefile ] && [ $biasnumerrors -eq 0 ] + if [ -e $biasvoltagefile ] then if [ $runnumber -gt 20120324 ] then - value="Uout" - else - value="U" - fi - getfitsstatistics $biasvoltagefile $value $tstart $tstop - if [ $good -eq 0 ] - then - getfitsstatistics $biasvoltagefile $value $tstart2 $tstop - fi - if [ $good -gt 0 ] - then - query=$query", fBiasVoltageMedian="$median + biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c Uout --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null` + evaluatestatistics "Uout" $biasstats + if [ "$evaluation" = "" ] + then + biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c Uout --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null` + evaluatestatistics "Uout" $biasstats + fi + else + biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c U --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null` + evaluatestatistics "U" $biasstats + if [ "$evaluation" = "" ] + then + biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c U --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null` + evaluatestatistics "U" $biasstats + fi + fi + if [ "$evaluation" != "" ] + then + query=$query", fBiasVoltageMedian="$med else query=$query", fBiasVoltageMedian=NULL"