Changeset 18135 for trunk/DataCheck


Ignore:
Timestamp:
02/18/15 09:26:41 (10 years ago)
Author:
Daniela Dorner
Message:
added mode to update aux data in case inserting in La Palma had a problem
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Processing/FillAuxData.sh

    r18083 r18135  
    77if [ "$doupdate" = "" ]
    88then
    9    doupdate="yes" # update all entries (needed when new fields have been added)
     9#   doupdate="yes" # update all entries (needed when new fields have been added)
     10#   doupdate="force" # needed when something with insert in La Palma did not work (adds more information)
    1011   doupdate="no" # fill only entries which are not yet existing (default)
    1112fi
    … …  
    178179      printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
    179180      echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
    180       #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`
    181       
     181      echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`
     182 
    182183      # get information from rawfile
    183184      rawfile=`ls $rawdir/$runnumber"_"\`printf %03d $filenum\`.fits*`
    … …  
    199200         continue
    200201      fi
     202      echo $runtype
    201203      # assuming that at least TSTARTI and TSTOPI are consistent
    202204      #echo $rawfile
    … …  
    241243               printprocesslog "WARN couldn't get sourcekey for source "$sourcename" from DB for "$runnumber"_"$filenum
    242244            fi
     245         fi
     246      fi
     247      if [ "$doupdate" == "force" ]
     248      then
     249         # set runtype to 'unknown', if no runtype could be retrieved from file
     250         if [ "$runtype" == "" ]
     251         then
     252            runtype="n/a"
     253         fi
     254         # on 15.11.2011 the runtypes had different names
     255         if [ "$date" == "2011/11/15" ]
     256         then
     257            if [ "$runtype" == "drs-calib" ]
     258            then
     259               runtype="drs-gain"
     260            fi
     261            if [ "$runtype" == "drs-time-calib" ]
     262            then
     263               runtype="drs-time"
     264            fi
     265            if [ "$runtype" == "pedestal" ]
     266            then
     267               runtype="drs-pedestal"
     268            fi
     269            if [ "$runtype" == "light-pulser" ]
     270            then
     271               runtype="light-pulser-ext"
     272            fi
     273            if [ "$runtype" == "pedestal-on" ]
     274            then
     275               runtype="pedestal"
     276            fi
     277         fi
     278         # get runtype
     279         query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
     280         result2=( `sendquery` )
     281         if [ ${#result2} -eq 0 ]
     282         then
     283            printprocesslog "ERROR "$numberfromname": Could not query fRunTypeKey for runtype "$runtype" ."
     284            continue
     285         fi
     286         # in newest data start time is in DATE-OBS
     287         # in older data start time is in TSTART
     288         # in the beginning TSTART was empty
     289         #runstart=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     290         runstart=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     291         #runstart2=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     292         runstart2=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     293         if [ "$runstart" == ""  ]
     294         then
     295            if [ "$runstart2" == ""  ]
     296            then
     297               #runstart=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     298               runstart=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     299            else
     300               runstart=$runstart2
     301            fi
     302         fi
     303         # in newest data start time is in DATE-END
     304         # in older data start time is in TSTOP
     305         # in the beginning TSTOP was empty
     306         #runstop=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     307         runstop=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     308         #runstop2=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     309         runstop2=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     310         if [ "$runstop" == ""  ]
     311         then
     312            if [ "$runstop2" == ""  ]
     313            then
     314               runstop=`stat $rawfile  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
     315            else
     316               runstop=$runstop2
     317            fi
     318         fi
     319         echo $runstart $runstop
     320         #numevents=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
     321         numevents=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
     322         #roi=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
     323         roi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
     324         #roitm=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
     325         roitm=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
     326         #numphys=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
     327         numphys=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
     328         #numext1=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
     329         numext1=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
     330         #numext2=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
     331         numext2=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
     332         #numelp=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
     333         numelp=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
     334         #numilp=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
     335         numilp=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
     336         #numoth=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
     337         numoth=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
     338         #numped=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
     339         numped=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
     340         #numtime=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
     341         numtime=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
     342         #compiled=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
     343         compiled=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
     344         if ! [ "$compiled" == "" ]
     345         then
     346            compiletime=`date +'%F %H:%M:%S' --date="${compiled}" `
     347         else
     348            compiletime=
     349         fi
     350         #revnum=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
     351         revnum=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
     352         # get checksums from header
     353         #checksum=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
     354         checksum=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
     355         if [ "$checksum" == "" ]
     356         then
     357            printprocesslog "WARN checksum for file "$rawfile" is empty."
     358         fi
     359         #datasum=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
     360         datasum=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
     361         if [ "$datasum" == "" ]
     362         then
     363            printprocesslog "WARN datasum for file "$rawfile" is empty."
     364         fi
     365         # check if this run has drs file
     366         #   in case file is available, get STEP from header
     367         # in the very beginning only drs-files were existing
     368         # in the beginning the keywords DRSCALIB and STEP were not existing
     369         drsfile=`echo $rawfile | sed -e 's/fits/drs.fits/'`
     370         numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
     371         #drscalib=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep DRSCALIB | grep -E -o "['][TF][']" | sed -e "s/'//g"`
     372         drscalib=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"`
     373         if [ "$drscalib" == "T" ]
     374         then
     375            #step=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep DRSSTEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
     376            step=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"`
     377            #stepfromdrs=`$factpath/fitsdump -h -t Events $drsfile  2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
     378            stepfromdrs=`$factpath/fitsdump -h $drsfile  2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"`
     379            if [ "$stepfromdrs" != "$step" ]
     380            then
     381               printprocesslog "ERROR for file "$rawfile" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree."
     382               if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ]
     383               then
     384                  step=$stepfromdrsfile
     385                  printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$rawfile"."
     386               fi
     387            fi
     388            if ! [ $numdrsfiles -eq 1 ]
     389            then
     390               printprocesslog "ERROR for file "$rawfile" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
     391            fi
     392            if [ "$step" = "" ]
     393            then
     394               printprocesslog "ERROR file "$rawfile" has drsfiles ("$numdrsfiles"), but step ("$step") is empty."
     395            fi
     396         else
     397            if ! [ "$drscalib" == "F" ]
     398            then
     399               printprocesslog "WARN for file "$rawfile" DRSCALIB is neither T nor F."
     400            fi
    243401         fi
    244402      fi
    … …  
    388546         query=$query", fBiasVoltageMedian=NULL"
    389547      fi
     548      if [ "$doupdate" == "force" ]
     549      then
     550         query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'"
     551         query=$query", fRunTypeKey="${result2[0]}
     552         if [ "$numevents" != "" ]
     553         then
     554            query=$query", fNumEvents="$numevents
     555         fi
     556         if [ "$roi" != "" ]
     557         then
     558            query=$query", fROI="$roi
     559         fi
     560         if [ "$roitm" != "" ]
     561         then
     562            query=$query", fROITimeMarker="$roitm
     563         fi
     564         if [ "$numphys" != "" ]
     565         then
     566            query=$query", fNumPhysicsTrigger="$numphys
     567         fi
     568         if [ "$numext1" != "" ]
     569         then
     570            query=$query", fNumExt1Trigger="$numext1
     571         fi
     572         if [ "$numext2" != "" ]
     573         then
     574            query=$query", fNumExt2Trigger="$numext2
     575         fi
     576         if [ "$numelp" != "" ]
     577         then
     578            query=$query", fNumELPTrigger="$numelp
     579         fi
     580         if [ "$numilp" != "" ]
     581         then
     582            query=$query", fNumILPTrigger="$numilp
     583         fi
     584         if [ "$numped" != "" ]
     585         then
     586            query=$query", fNumPedestalTrigger="$numped
     587         fi
     588         if [ "$numtime" != "" ]
     589         then
     590            query=$query", fNumTimeTrigger="$numtime
     591         fi
     592         if [ "$numoth" != "" ]
     593         then
     594            query=$query", fNumOtherTrigger="$numoth
     595         fi
     596         if [ "$checksum" != "" ]
     597         then
     598            query=$query", fCheckSum='"$checksum"'"
     599         fi
     600         if [ "$datasum" != "" ]
     601         then
     602            query=$query", fDataSum='"$datasum"'"
     603         fi
     604         if [ "$numdrsfiles" != "" ]
     605         then
     606            query=$query", fHasDrsFile="$numdrsfiles
     607         fi
     608         if [ "$step" != "" ]
     609         then
     610            query=$query", fDrsStep="$step
     611         fi
     612         if [ "$compiletime" != "" ]
     613         then
     614            query=$query", fCompileTime='"$compiletime"'"
     615         fi
     616         if [ "$revnum" != "" ]
     617         then
     618            query=$query", fRevisionNumber='"$revnum"'"
     619         fi
     620      fi
     621
    390622     
    391623      # add where condition
    392624      query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
    393       #echo $query
     625      echo $query
    394626
    395627      # send query to DB
Note: See TracChangeset for help on using the changeset viewer.