Changeset 12632 for trunk/DataCheck


Ignore:
Timestamp:
11/24/11 08:38:47 (13 years ago)
Author:
Daniela Dorner
Message:
added new values, rearranged checks
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/CheckRawData.sh

    r12623 r12632  
    3030# get last 3 nights
    3131dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` )
    32 dates=( `date +%Y/%m/%d --date="-3day"` )
     32dates=( `date +%Y/%m/%d --date="-6day"` )
     33dates=( "2011/11/23" )
    3334
    3435# do rsync for rawfiles of these dates
    … …  
    9495      fi
    9596     
    96       # get and check run and file number
     97      # get run and file number form filename
    9798      runnumbererror="no"
    9899      numbererror="no"
    99100      numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
    100       runnumberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
    101       runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
    102       filenumberfromfileorig=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
    103       filenumberfromfile=`printf %03d $filenumberfromfileorig`
    104       numberfromfile=$runnumberfromfile"_"$filenumberfromfile
     101      runnumberfromname=`echo $numberfromname | cut -d_ -f1`
     102      filenumberfromname=`echo $numberfromname | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'`
    105103      if [ "$runnumber" != "$runnumberfromname" ]
    106104      then
    107105         runnumbererror="yes"
    108       fi
    109       if [ "$numberfromfile" != "$numberfromname" ]
    110       then
    111          numbererror="yes"
     106         echo "ERROR: for file "$file" runnumber from date and filename don't agree ("$runnumber" - "$runnumberfromname")"
     107         echo "ERROR: for file "$file" runnumber from date and filename don't agree ("$runnumber" - "$runnumberfromname")" >> $logfile 2>&1
    112108      fi
    113109     
    114110      # check if entry already exists
    115       query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
     111      query3="SELECT fRunNumber FROM RunInfo WHERE Concat(fRunNumber, '_', LPAD(fFileNumber, 3, 0))='"$numberfromname"'"
    116112      echo "Q3:"$query3 >> $logfile 2>&1 2>&1
    117       result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
     113      result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3" 2>> $logfile`
    118114
    119115      # only proceed with file
    … …  
    127123     
    128124      # check if entry has already checksums
    129       query5="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
     125      query5="SELECT fRunNumber FROM RunInfo WHERE Concat(fRunNumber, '_', LPAD(fFileNumber, 3, 0))='"$numberfromname"'"
    130126      query5=$query5" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
    131127      echo "Q5:"$query5 >> $logfile 2>&1
    132       result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"`
     128      result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5" 2>> $logfile`
    133129
    134130      # get md5sums of raw and zip file
    … …  
    154150         fi
    155151      fi
    156      
    157       # check if this run has drs file
    158       drsfile=`echo $file | sed -e 's/fits/drs.fits/'`
    159       numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
    160      
    161       # get other variables from header
    162       runtype=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
    163       roi=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
    164       numevents=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
    165       # in newest data start time is in DATE-OBS
    166       # in older data start time is in TSTART
    167       # in the beginning TSTART was empty
    168       runstart=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    169       runstart2=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    170       if [ "$runstart" == ""  ]
    171       then
    172          if [ "$runstart2" == ""  ]
    173          then
    174             runstart=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    175          else
    176             runstart=$runstart2
    177          fi
    178       fi
    179       # in newest data start time is in DATE-END
    180       # in older data start time is in TSTOP
    181       # in the beginning TSTOP was empty
    182       runstop=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    183       runstop2=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
    184       if [ "$runstop" == ""  ]
    185       then
    186          if [ "$runstop2" == ""  ]
    187          then
    188             runstop=`stat $file  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
    189          else
    190             runstop=$runstop2
    191          fi
    192       fi
    193 
     152
     153      runtype=
    194154      # check if fits file is corrupted
    195155      numfitserrors=0
    196       checkfitsfile=`fverify $file | grep '0 error(s)'`
     156      checkfitsfile=`fverify $file  2>> $logfile | grep '0 error(s)'`
    197157      if [ "$checkfitsfile" == "" ]
    198158      then
    199159         numfitserrors=1
     160      fi
     161
     162      if [ $numfitserrors -eq 0 ]
     163      then
     164         # get run and file number from file
     165         runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
     166         filenumberfromfileorig=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
     167         numberfromfile=$runnumberfromfile"_"`printf %03d $filenumberfromfileorig`
     168         # compare numbers
     169         if [ "$numberfromfile" != "$numberfromname" ]
     170         then
     171            numbererror="yes"
     172            echo "ERROR: for file "$file" number from filename and file don't agree ("$numberfromname" -"$numberfromfile")"
     173            echo "ERROR: for file "$file" number from filename and file don't agree ("$numberfromname" -"$numberfromfile")" >> $logfile 2>&1
     174         fi
     175     
     176         # check if this run has drs file
     177         drsfile=`echo $file | sed -e 's/fits/drs.fits/'`
     178         numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
     179         
     180         # get other variables from header
     181         runtype=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
     182         roi=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
     183         numevents=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
     184         numphys=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
     185         numext1=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
     186         numext2=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
     187         numelp=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
     188         numilp=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
     189         numoth=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
     190         numped=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
     191         numtime=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
     192         # in newest data start time is in DATE-OBS
     193         # in older data start time is in TSTART
     194         # in the beginning TSTART was empty
     195         runstart=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     196         runstart2=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     197         if [ "$runstart" == ""  ]
     198         then
     199            if [ "$runstart2" == ""  ]
     200            then
     201               runstart=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     202            else
     203               runstart=$runstart2
     204            fi
     205         fi
     206         # in newest data start time is in DATE-END
     207         # in older data start time is in TSTOP
     208         # in the beginning TSTOP was empty
     209         runstop=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     210         runstop2=`/home/fact/FACT++/fitsdump -h -t Events $file  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
     211         if [ "$runstop" == ""  ]
     212         then
     213            if [ "$runstop2" == ""  ]
     214            then
     215               runstop=`stat $file  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
     216            else
     217               runstop=$runstop2
     218            fi
     219         fi
     220      fi
     221     
     222      # set runtype to 'unknown' if no runtype could be queried
     223      if [ "$runtype" == "" ]
     224      then
     225         runtype="n/a"
     226      fi
     227      # on 15.11.2011 the runtypes had different names
     228      if [ "$date" == "2011/11/15" ]
     229      then
     230         if [ "$runtype" == "drs-calib" ]
     231         then
     232            runtype="drs-gain"
     233         fi
     234         if [ "$runtype" == "drs-time-calib" ]
     235         then
     236            runtype="drs-time"
     237         fi
     238         if [ "$runtype" == "pedestal" ]
     239         then
     240            runtype="drs-pedestal"
     241         fi
     242         if [ "$runtype" == "light-pulser" ]
     243         then
     244            runtype="light-pulser-ext"
     245         fi
     246         if [ "$runtype" == "pedestal-on" ]
     247         then
     248            runtype="pedestal"
     249         fi
    200250      fi
    201251      # get runtype
    202252      query2="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
    203253      echo "Q2:"$query2 >> $logfile 2>&1
    204       result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` )
     254      result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2" 2>> $logfile` )
    205255      if [ ${#result2} -eq 0 ]
    206256      then
    207          echo "Could not query fRunTypeKey for runtype "$runtype
    208          echo "Could not query fRunTypeKey for runtype "$runtype >> $logfile 2>&1
     257         echo $numberfromname": Could not query fRunTypeKey for runtype "$runtype
     258         echo $numberfromname": Could not query fRunTypeKey for runtype "$runtype >> $logfile 2>&1
    209259         exit
    210260      fi
     261
    211262      # insert or update depending on whether run exists
    212263      if [ "$result3" == "" ]
    213264      then
    214265         query4="INSERT"
    215          querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromfileorig", "
     266         querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromname", "
    216267         querystop=""
    217268      else
    218269         query4="UPDATE"
    219          querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig
    220       fi
    221       query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}", fNumEvents="$numevents", fROI="$roi
     270         querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromname
     271      fi
     272      query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}
     273      if [ $numfitserrors -eq 0 ]
     274      then
     275         query4=$query4", fNumEvents="$numevents", fROI="$roi
     276         query4=$query4", fRunStart='"$runstart"', fRunStop='"$runstop"'"
     277         if [ "$numphys" != "" ]
     278         then
     279            query4=$query4", fNumPhysicsTrigger="$numphys
     280         fi
     281         if [ "$numext1" != "" ]
     282         then
     283            query4=$query4", fNumExt1Trigger="$numext1
     284         fi
     285         if [ "$numext2" != "" ]
     286         then
     287            query4=$query4", fNumExt2Trigger="$numext2
     288         fi
     289         if [ "$numelp" != "" ]
     290         then
     291            query4=$query4", fNumELPTrigger="$numelp
     292         fi
     293         if [ "$numilp" != "" ]
     294         then
     295            query4=$query4", fNumILPTrigger="$numilp
     296         fi
     297         if [ "$numped" != "" ]
     298         then
     299            query4=$query4", fNumPedestalTrigger="$numped
     300         fi
     301         if [ "$numtime" != "" ]
     302         then
     303            query4=$query4", fNumTimeTrigger="$numtime
     304         fi
     305         if [ "$numoth" != "" ]
     306         then
     307            query4=$query4", fNumOtherTrigger="$numoth
     308         fi
     309      fi
    222310      if [ "$md5sum" != "" ]
    223311      then
    224312         query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
    225       fi
    226       query4=$query4", fHasDrsFile="$numdrsfiles
     313         query4=$query4", fHasDrsFile="$numdrsfiles
     314      fi
    227315      query4=$query4", fFitsFileErrors="$numfitserrors
    228       query4=$query4", fRunStart='"$runstart"', fRunStop='"$runstop"'"
    229316      query4=$query4" "$querystop
    230317      echo "Q4:"$query4 >> $logfile 2>&1
    231318      if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query4" >> $logfile 2>&1
    232319      then
    233          echo "insert/update of "$numfromfile" to mysql" >> $logfile 2>&1
     320         echo "insert/update of "$numfromfile" to mysql failed" >> $logfile 2>&1
    234321      fi
    235322
Note: See TracChangeset for help on using the changeset viewer.