Changeset 12933 for trunk/DataCheck


Ignore:
Timestamp:
02/23/12 16:49:03 (13 years ago)
Author:
Daniela Dorner
Message:
added new variables and further checks
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/CheckRawData.sh

    r12788 r12933  
    66# options:
    77skipmd5sum="no" # fill md5 sums in any case
    8 skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db
     8skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
    99#skipmd5sum="yes" # do not fill md5 sums in any case
    1010
    1111doupdate="yes" # update all entries
    12 doupdate="no" # fill only entries which are not yet existing
     12#doupdate="no" # fill only entries which are not yet existing #default
    1313
    1414source `dirname $0`/Sourcefile.sh
    … …  
    3737dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` )
    3838#dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw
     39#dates=( "2012/02/19" )
    3940
    4041# do check for rawfiles of these dates
    … …  
    6263   do
    6364      printprocesslog "INFO checking file "$file
     65      #echo "INFO checking file "$file
    6466
    6567      # raw and original file
    … …  
    115117     
    116118      # check if entry already exists
    117       query3="SELECT fRunNumber FROM RunInfo WHERE Concat(fRunNumber, '_', LPAD(fFileNumber, 3, 0))='"$numberfromname"'"
     119      query3="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
    118120      printprocesslog "DEBUG check if entry already exists in DB. QUERY: "$query3
    119121      result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
    … …  
    131133      # check if fits file is corrupted
    132134      numfitserrors=0
    133       checkfitsfile=`fverify $rawfile | grep '0 error(s)'`
    134       if [ "$checkfitsfile" == "" ]
    135       then
    136          numfitserrors=1
     135      #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
     136      numfitserrors=`fverify $rawfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
     137      #if [ "$checkfitsfile" == "" ]
     138      if [ $numfitserrors -gt 0 ]
     139      then
     140         printprocesslog "WARN "$rawfile" has "$numfitserrors" fitserror(s). "
     141         #numfitserrors=1
    137142      fi
    138143
    … …  
    158163         # in the very beginning only drs-files were existing
    159164         # in the beginning the keywords DRSCALIB and STEP were not existing
     165         numdrsfiles=
    160166         step=
    161167         drsfile=`echo $file | sed -e 's/fits/drs.fits/'`
    … …  
    168174            then
    169175               printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
     176            fi
     177         else
     178            if ! [ "$drscalib" == "F" ]
     179            then
     180               printprocesslog "WARN for file "$file" DRSCALIB is neither T nor F."
    170181            fi
    171182         fi
    … …  
    185196         numped=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
    186197         numtime=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
     198         compiled=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
     199         if ! [ "$compiled" == "" ]
     200         then
     201            compiletime=`date +'%F %H:%M:%S' --date="${compiled}" `
     202         else
     203            compiletime=
     204         fi
     205         revnum=`$factpath/fitsdump -h -t Events $file  2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
    187206         # in newest data start time is in DATE-OBS
    188207         # in older data start time is in TSTART
    … …  
    255274
    256275      # check if entry has already checksums
    257       query5="SELECT fRunNumber FROM RunInfo WHERE Concat(fRunNumber, '_', LPAD(fFileNumber, 3, 0))='"$numberfromname"'"
     276      query5="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
    258277      query5=$query5" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
    259278      printprocesslog "DEBUG check if md5sums are alreay in DB. QUERY:"$query5
    … …  
    287306      then
    288307         query4="INSERT"
    289          querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromname", "
     308         querymid=" fNight="$runnumber", fRunID="$filenumberfromname", "
    290309         querystop=
    291310      else
    292311         query4="UPDATE"
    293312         querymid=
    294          querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromname
     313         querystop=" WHERE fNight="$runnumber" AND fRunID="$filenumberfromname
    295314      fi
    296315      query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}
    … …  
    300319         if [ "$numevents" != "" ]
    301320         then
    302             query4=$query4", fNumEvents="$numevent
     321            query4=$query4", fNumEvents="$numevents
    303322         fi
    304323         if [ "$roi" != "" ]
    … …  
    346365      then
    347366         query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
     367      fi
     368      if [ "$numdrsfiles" != "" ]
     369      then
    348370         query4=$query4", fHasDrsFile="$numdrsfiles
    349371      fi
    … …  
    351373      then
    352374         query4=$query4", fDrsStep="$step
     375      fi
     376      if [ "$compiletime" != "" ]
     377      then
     378         query4=$query4", fCompileTime='"$compiletime"'"
     379      fi
     380      if [ "$revnum" != "" ]
     381      then
     382         query4=$query4", fRevisionNumber='"$revnum"'"
    353383      fi
    354384      query4=$query4", fFitsFileErrors="$numfitserrors
Note: See TracChangeset for help on using the changeset viewer.