Index: /trunk/DataCheck/CheckRawData.sh =================================================================== --- /trunk/DataCheck/CheckRawData.sh (revision 12631) +++ /trunk/DataCheck/CheckRawData.sh (revision 12632) @@ -30,5 +30,6 @@ # get last 3 nights dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` ) -dates=( `date +%Y/%m/%d --date="-3day"` ) +dates=( `date +%Y/%m/%d --date="-6day"` ) +dates=( "2011/11/23" ) # do rsync for rawfiles of these dates @@ -94,26 +95,21 @@ fi - # get and check run and file number + # get run and file number form filename runnumbererror="no" numbererror="no" numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'` - runnumberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'` - runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'` - filenumberfromfileorig=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'` - filenumberfromfile=`printf %03d $filenumberfromfileorig` - numberfromfile=$runnumberfromfile"_"$filenumberfromfile + runnumberfromname=`echo $numberfromname | cut -d_ -f1` + filenumberfromname=`echo $numberfromname | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'` if [ "$runnumber" != "$runnumberfromname" ] then runnumbererror="yes" - fi - if [ "$numberfromfile" != "$numberfromname" ] - then - numbererror="yes" + echo "ERROR: for file "$file" runnumber from date and filename don't agree ("$runnumber" - "$runnumberfromname")" + echo "ERROR: for file "$file" runnumber from date and filename don't agree ("$runnumber" - "$runnumberfromname")" >> $logfile 2>&1 fi # check if entry already exists - query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig + query3="SELECT fRunNumber FROM RunInfo WHERE Concat(fRunNumber, '_', LPAD(fFileNumber, 3, 0))='"$numberfromname"'" echo "Q3:"$query3 >> $logfile 2>&1 2>&1 - result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"` + result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3" 2>> $logfile` # only proceed with file @@ -127,8 +123,8 @@ # check if entry has already checksums - query5="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig + query5="SELECT fRunNumber FROM RunInfo WHERE Concat(fRunNumber, '_', LPAD(fFileNumber, 3, 0))='"$numberfromname"'" query5=$query5" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)" echo "Q5:"$query5 >> $logfile 2>&1 - result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"` + result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5" 2>> $logfile` # get md5sums of raw and zip file @@ -154,82 +150,173 @@ fi fi - - # check if this run has drs file - drsfile=`echo $file | sed -e 's/fits/drs.fits/'` - numdrsfiles=`ls $drsfile 2>/dev/null | wc -l` - - # get other variables from header - runtype=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` - roi=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` - numevents=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` - # in newest data start time is in DATE-OBS - # in older data start time is in TSTART - # in the beginning TSTART was empty - runstart=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - runstart2=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - if [ "$runstart" == "" ] - then - if [ "$runstart2" == "" ] - then - runstart=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - else - runstart=$runstart2 - fi - fi - # in newest data start time is in DATE-END - # in older data start time is in TSTOP - # in the beginning TSTOP was empty - runstop=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - runstop2=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` - if [ "$runstop" == "" ] - then - if [ "$runstop2" == "" ] - then - runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` - else - runstop=$runstop2 - fi - fi - + + runtype= # check if fits file is corrupted numfitserrors=0 - checkfitsfile=`fverify $file | grep '0 error(s)'` + checkfitsfile=`fverify $file 2>> $logfile | grep '0 error(s)'` if [ "$checkfitsfile" == "" ] then numfitserrors=1 + fi + + if [ $numfitserrors -eq 0 ] + then + # get run and file number from file + runnumberfromfile=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'` + filenumberfromfileorig=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'` + numberfromfile=$runnumberfromfile"_"`printf %03d $filenumberfromfileorig` + # compare numbers + if [ "$numberfromfile" != "$numberfromname" ] + then + numbererror="yes" + echo "ERROR: for file "$file" number from filename and file don't agree ("$numberfromname" -"$numberfromfile")" + echo "ERROR: for file "$file" number from filename and file don't agree ("$numberfromname" -"$numberfromfile")" >> $logfile 2>&1 + fi + + # check if this run has drs file + drsfile=`echo $file | sed -e 's/fits/drs.fits/'` + numdrsfiles=`ls $drsfile 2>/dev/null | wc -l` + + # get other variables from header + runtype=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` + roi=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` + numevents=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` + numphys=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'` + numext1=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` + numext2=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'` + numelp=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'` + numilp=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'` + numoth=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'` + numped=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'` + numtime=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'` + # in newest data start time is in DATE-OBS + # in older data start time is in TSTART + # in the beginning TSTART was empty + runstart=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstart2=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstart" == "" ] + then + if [ "$runstart2" == "" ] + then + runstart=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + else + runstart=$runstart2 + fi + fi + # in newest data start time is in DATE-END + # in older data start time is in TSTOP + # in the beginning TSTOP was empty + runstop=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstop2=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstop" == "" ] + then + if [ "$runstop2" == "" ] + then + runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` + else + runstop=$runstop2 + fi + fi + fi + + # set runtype to 'unknown' if no runtype could be queried + if [ "$runtype" == "" ] + then + runtype="n/a" + fi + # on 15.11.2011 the runtypes had different names + if [ "$date" == "2011/11/15" ] + then + if [ "$runtype" == "drs-calib" ] + then + runtype="drs-gain" + fi + if [ "$runtype" == "drs-time-calib" ] + then + runtype="drs-time" + fi + if [ "$runtype" == "pedestal" ] + then + runtype="drs-pedestal" + fi + if [ "$runtype" == "light-pulser" ] + then + runtype="light-pulser-ext" + fi + if [ "$runtype" == "pedestal-on" ] + then + runtype="pedestal" + fi fi # get runtype query2="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'" echo "Q2:"$query2 >> $logfile 2>&1 - result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` ) + result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2" 2>> $logfile` ) if [ ${#result2} -eq 0 ] then - echo "Could not query fRunTypeKey for runtype "$runtype - echo "Could not query fRunTypeKey for runtype "$runtype >> $logfile 2>&1 + echo $numberfromname": Could not query fRunTypeKey for runtype "$runtype + echo $numberfromname": Could not query fRunTypeKey for runtype "$runtype >> $logfile 2>&1 exit fi + # insert or update depending on whether run exists if [ "$result3" == "" ] then query4="INSERT" - querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromfileorig", " + querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromname", " querystop="" else query4="UPDATE" - querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig - fi - query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}", fNumEvents="$numevents", fROI="$roi + querystop=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromname + fi + query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]} + if [ $numfitserrors -eq 0 ] + then + query4=$query4", fNumEvents="$numevents", fROI="$roi + query4=$query4", fRunStart='"$runstart"', fRunStop='"$runstop"'" + if [ "$numphys" != "" ] + then + query4=$query4", fNumPhysicsTrigger="$numphys + fi + if [ "$numext1" != "" ] + then + query4=$query4", fNumExt1Trigger="$numext1 + fi + if [ "$numext2" != "" ] + then + query4=$query4", fNumExt2Trigger="$numext2 + fi + if [ "$numelp" != "" ] + then + query4=$query4", fNumELPTrigger="$numelp + fi + if [ "$numilp" != "" ] + then + query4=$query4", fNumILPTrigger="$numilp + fi + if [ "$numped" != "" ] + then + query4=$query4", fNumPedestalTrigger="$numped + fi + if [ "$numtime" != "" ] + then + query4=$query4", fNumTimeTrigger="$numtime + fi + if [ "$numoth" != "" ] + then + query4=$query4", fNumOtherTrigger="$numoth + fi + fi if [ "$md5sum" != "" ] then query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'" - fi - query4=$query4", fHasDrsFile="$numdrsfiles + query4=$query4", fHasDrsFile="$numdrsfiles + fi query4=$query4", fFitsFileErrors="$numfitserrors - query4=$query4", fRunStart='"$runstart"', fRunStop='"$runstop"'" query4=$query4" "$querystop echo "Q4:"$query4 >> $logfile 2>&1 if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query4" >> $logfile 2>&1 then - echo "insert/update of "$numfromfile" to mysql" >> $logfile 2>&1 + echo "insert/update of "$numfromfile" to mysql failed" >> $logfile 2>&1 fi