Index: trunk/DataCheck/Processing/FillAuxData.sh =================================================================== --- trunk/DataCheck/Processing/FillAuxData.sh (revision 18093) +++ trunk/DataCheck/Processing/FillAuxData.sh (revision 18135) @@ -7,5 +7,6 @@ if [ "$doupdate" = "" ] then - doupdate="yes" # update all entries (needed when new fields have been added) +# doupdate="yes" # update all entries (needed when new fields have been added) +# doupdate="force" # needed when something with insert in La Palma did not work (adds more information) doupdate="no" # fill only entries which are not yet existing (default) fi @@ -178,6 +179,6 @@ printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum` echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1 - #echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` - + echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` + # get information from rawfile rawfile=`ls $rawdir/$runnumber"_"\`printf %03d $filenum\`.fits*` @@ -199,4 +200,5 @@ continue fi + echo $runtype # assuming that at least TSTARTI and TSTOPI are consistent #echo $rawfile @@ -241,4 +243,160 @@ printprocesslog "WARN couldn't get sourcekey for source "$sourcename" from DB for "$runnumber"_"$filenum fi + fi + fi + if [ "$doupdate" == "force" ] + then + # set runtype to 'unknown', if no runtype could be retrieved from file + if [ "$runtype" == "" ] + then + runtype="n/a" + fi + # on 15.11.2011 the runtypes had different names + if [ "$date" == "2011/11/15" ] + then + if [ "$runtype" == "drs-calib" ] + then + runtype="drs-gain" + fi + if [ "$runtype" == "drs-time-calib" ] + then + runtype="drs-time" + fi + if [ "$runtype" == "pedestal" ] + then + runtype="drs-pedestal" + fi + if [ "$runtype" == "light-pulser" ] + then + runtype="light-pulser-ext" + fi + if [ "$runtype" == "pedestal-on" ] + then + runtype="pedestal" + fi + fi + # get runtype + query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'" + result2=( `sendquery` ) + if [ ${#result2} -eq 0 ] + then + printprocesslog "ERROR "$numberfromname": Could not query fRunTypeKey for runtype "$runtype" ." + continue + fi + # in newest data start time is in DATE-OBS + # in older data start time is in TSTART + # in the beginning TSTART was empty + #runstart=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstart=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + #runstart2=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstart2=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstart" == "" ] + then + if [ "$runstart2" == "" ] + then + #runstart=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstart=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + else + runstart=$runstart2 + fi + fi + # in newest data start time is in DATE-END + # in older data start time is in TSTOP + # in the beginning TSTOP was empty + #runstop=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstop=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + #runstop2=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstop2=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstop" == "" ] + then + if [ "$runstop2" == "" ] + then + runstop=`stat $rawfile 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` + else + runstop=$runstop2 + fi + fi + echo $runstart $runstop + #numevents=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep Events | grep -E -o '[0-9]+'` + numevents=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep Events | grep -E -o '[0-9]+'` + #roi=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` + roi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` + #roitm=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'` + roitm=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'` + #numphys=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'` + numphys=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'` + #numext1=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` + numext1=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` + #numext2=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` + numext2=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` + #numelp=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'` + numelp=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'` + #numilp=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'` + numilp=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'` + #numoth=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'` + numoth=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'` + #numped=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'` + numped=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'` + #numtime=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'` + numtime=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'` + #compiled=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"` + compiled=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"` + if ! [ "$compiled" == "" ] + then + compiletime=`date +'%F %H:%M:%S' --date="${compiled}" ` + else + compiletime= + fi + #revnum=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"` + revnum=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"` + # get checksums from header + #checksum=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'` + checksum=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'` + if [ "$checksum" == "" ] + then + printprocesslog "WARN checksum for file "$rawfile" is empty." + fi + #datasum=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'` + datasum=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'` + if [ "$datasum" == "" ] + then + printprocesslog "WARN datasum for file "$rawfile" is empty." + fi + # check if this run has drs file + # in case file is available, get STEP from header + # in the very beginning only drs-files were existing + # in the beginning the keywords DRSCALIB and STEP were not existing + drsfile=`echo $rawfile | sed -e 's/fits/drs.fits/'` + numdrsfiles=`ls $drsfile 2>/dev/null | wc -l` + #drscalib=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep DRSCALIB | grep -E -o "['][TF][']" | sed -e "s/'//g"` + drscalib=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"` + if [ "$drscalib" == "T" ] + then + #step=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep DRSSTEP | grep -E -o "['][012][']" | sed -e "s/'//g"` + step=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"` + #stepfromdrs=`$factpath/fitsdump -h -t Events $drsfile 2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"` + stepfromdrs=`$factpath/fitsdump -h $drsfile 2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"` + if [ "$stepfromdrs" != "$step" ] + then + printprocesslog "ERROR for file "$rawfile" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree." + if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ] + then + step=$stepfromdrsfile + printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$rawfile"." + fi + fi + if ! [ $numdrsfiles -eq 1 ] + then + printprocesslog "ERROR for file "$rawfile" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree." + fi + if [ "$step" = "" ] + then + printprocesslog "ERROR file "$rawfile" has drsfiles ("$numdrsfiles"), but step ("$step") is empty." + fi + else + if ! [ "$drscalib" == "F" ] + then + printprocesslog "WARN for file "$rawfile" DRSCALIB is neither T nor F." + fi fi fi @@ -388,8 +546,82 @@ query=$query", fBiasVoltageMedian=NULL" fi + if [ "$doupdate" == "force" ] + then + query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'" + query=$query", fRunTypeKey="${result2[0]} + if [ "$numevents" != "" ] + then + query=$query", fNumEvents="$numevents + fi + if [ "$roi" != "" ] + then + query=$query", fROI="$roi + fi + if [ "$roitm" != "" ] + then + query=$query", fROITimeMarker="$roitm + fi + if [ "$numphys" != "" ] + then + query=$query", fNumPhysicsTrigger="$numphys + fi + if [ "$numext1" != "" ] + then + query=$query", fNumExt1Trigger="$numext1 + fi + if [ "$numext2" != "" ] + then + query=$query", fNumExt2Trigger="$numext2 + fi + if [ "$numelp" != "" ] + then + query=$query", fNumELPTrigger="$numelp + fi + if [ "$numilp" != "" ] + then + query=$query", fNumILPTrigger="$numilp + fi + if [ "$numped" != "" ] + then + query=$query", fNumPedestalTrigger="$numped + fi + if [ "$numtime" != "" ] + then + query=$query", fNumTimeTrigger="$numtime + fi + if [ "$numoth" != "" ] + then + query=$query", fNumOtherTrigger="$numoth + fi + if [ "$checksum" != "" ] + then + query=$query", fCheckSum='"$checksum"'" + fi + if [ "$datasum" != "" ] + then + query=$query", fDataSum='"$datasum"'" + fi + if [ "$numdrsfiles" != "" ] + then + query=$query", fHasDrsFile="$numdrsfiles + fi + if [ "$step" != "" ] + then + query=$query", fDrsStep="$step + fi + if [ "$compiletime" != "" ] + then + query=$query", fCompileTime='"$compiletime"'" + fi + if [ "$revnum" != "" ] + then + query=$query", fRevisionNumber='"$revnum"'" + fi + fi + # add where condition query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum - #echo $query + echo $query # send query to DB