Index: trunk/DataCheck/CheckRawData.sh =================================================================== --- trunk/DataCheck/CheckRawData.sh (revision 12584) +++ trunk/DataCheck/CheckRawData.sh (revision 12609) @@ -3,4 +3,18 @@ today=`date +%F` logfile=/home/`whoami`/DataCheck/log/CheckRaw$today.log + +skipmd5sum="no" # fill md5 sums in any case +skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db +#skipmd5sum="yes" # do not fill md5 sums in any case + +doupdate="yes" # update all entries +#doupdate="no" # fill only entries which are not yet existing + +echo "" >> $logfile 2>&1 +echo "" >> $logfile 2>&1 +echo "" >> $logfile 2>&1 +echo `date`"executing "$0"..." >> $logfile 2>&1 +echo "=====> doupdate: "$doupdate >> $logfile 2>&1 +echo "=====> skipmd5sum: "$skipmd5sum >> $logfile 2>&1 password=`cat /home/fact/DataCheck/.pw` @@ -16,5 +30,5 @@ # get last 3 nights dates=( `date +%Y/%m/%d` `date +%Y/%m/%d --date="-1day"` `date +%Y/%m/%d --date="-2day"` `date +%Y/%m/%d --date="-3day"` ) -dates=( `date +%Y/%m/%d --date="-1day"` ) +dates=( `date +%Y/%m/%d --date="-2day"` ) # do rsync for rawfiles of these dates @@ -26,25 +40,27 @@ rawdir=/loc_data/raw/$date runnumber=`echo $date | sed -e 's/\///g'` - echo `date`": processing files in "$rawdir >> $logfile + echo `date`": processing files in "$rawdir >> $logfile 2>&1 # check if data are available from that night if ! [ -d $rawdir ] then - echo `date`": no data available in "$rawdir >> $logfile + echo `date`": no data available in "$rawdir >> $logfile 2>&1 continue fi # find all fits-files starting with the oldest file - echo `date`": finding files to be zipped in $rawdir..." >> $logfile + echo `date`": finding files to be zipped in $rawdir..." >> $logfile 2>&1 fitsfiles=`find $rawdir -type f -name '*.fits'| sort ` # loop to zip files - echo `date`": zipping files in $rawdir..." >> $logfile + echo `date`": checking files in $rawdir..." >> $logfile 2>&1 for file in $fitsfiles do - # check if raw file was accessed in the last 30 minutes - isnew=`find $file -amin -30` + echo "" >> $logfile 2>&1 + echo "checking file "$file >> $logfile 2>&1 + # check if raw file was changed in the last 30 minutes + isnew=`find $file -cmin -30` if [ "$isnew" != "" ] then - echo $file" is not older than 30 min => continue" >> $logfile + echo $file" is not older than 30 min => continue" >> $logfile 2>&1 continue fi @@ -54,5 +70,5 @@ if [ "$isnew" != "" ] then - echo $file" is a drs file => continue" >> $logfile + echo $file" is a drs file => continue" >> $logfile 2>&1 continue fi @@ -70,16 +86,11 @@ then # if times are not the same, the file is still open => no zip - echo `date`": file "$file" not yet closed => continue" >> $logfile + echo `date`": file "$file" not yet closed => continue" >> $logfile 2>&1 continue fi else # if the origfile doesn't exist, the data was probably written not on daq but on data - echo `date`": file "$file" was probably taken on data and not daq " >> $logfile - fi - - # get md5sum - md5sum=`md5sum $file | cut -d' ' -f1` - zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` - md5sumzip=`md5sum $zipfile | cut -d' ' -f1` + echo `date`": file "$file" was probably taken on data and not daq " >> $logfile 2>&1 + fi # get and check run and file number @@ -101,43 +112,85 @@ fi + # check if entry already exists + query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig + echo "Q3:"$query3 >> $logfile 2>&1 2>&1 + result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"` + + # only proceed with file + # if information is not yet in database + # and no update is wished ($doupdate) + if [ "$result3" != "" ] && [ "$doupdate" == "no" ] + then + echo $file" has been inserted already => continue " >> $logfile 2>&1 + continue + fi + + # check if entry has already checksums + query5="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig + query5=$query5" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)" + echo "Q5:"$query5 >> $logfile 2>&1 2>&1 + result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"` + + # get md5sums of raw and zip file + # to safe time for tests and update this can be skipped ($skipmd5sum) + md5sum= + md5sumzip= + if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ] + then + #echo "skip: "$skipmd5sum >> $logfile 2>&1 + #echo "res5: -"$result5"-" >> $logfile 2>&1 + #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ] + if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ] + then + echo "calculation md5sum for raw..." >> $logfile 2>&1 + md5sum=`md5sum $file | cut -d' ' -f1` + zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'` + echo "calculation md5sum for zipraw..." >> $logfile 2>&1 + md5sumzip=`md5sum $zipfile | cut -d' ' -f1` + fi + fi + + # check if this run has drs file + drsfile=`echo $file | sed -e 's/fits/drs.fits/'` + numdrsfiles=`ls $drsfile 2>/dev/null | wc -l` + # get other variables from header runtype=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` roi=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` numevents=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'` + runstart=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstop=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstart" == "" ] + then + runstart=`/home/fact/FACT++/fitsdump -h -t Events $file 2>/dev/null | grep DATE | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + fi + if [ "$runstop" == "" ] + then + runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` + fi # check if fits file is corrupted - fitsfileerror="no" + numfitserrors=0 checkfitsfile=`fverify $file | grep '0 error(s)'` if [ "$checkfitsfile" == "" ] then - fitsfileerror="yes" - fi - echo "summary for file"$file - echo " errors: "$fitsfileerror - echo " runnumber error: "$runnumbererror - echo " number error: "$numbererror - echo " roi: "$roi - echo " runtype: "$runtype - echo " numevents: "$numevents - echo " md5sum: "$md5sum - echo " md5sum(zip): "$md5sumzip - # check wehter entry has been made (status table) + numfitserrors=1 + fi # get runtype query2="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'" - echo "Q2:"$query2 + echo "Q2:"$query2 >> $logfile 2>&1 result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` ) if [ ${#result2} -eq 0 ] then echo "Could not query fRunTypeKey for runtype "$runtype + echo "Could not query fRunTypeKey for runtype "$runtype >> $logfile 2>&1 exit fi - # check if entry already exists - query3="SELECT fRunNumber FROM RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenumberfromfileorig - echo "Q3:"$query3 - resul3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"` + # insert or update depending on whether run exists if [ "$result3" == "" ] then query4="INSERT" querymid=" fRunNumber="$runnumber", fFileNumber="$filenumberfromfileorig", " + querystop="" else query4="UPDATE" @@ -145,11 +198,36 @@ fi query4=$query4" RunInfo SET "$querymid" fRunTypeKey="${result2[1]}", fNumEvents="$numevents", fROI="$roi - query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'" - echo "Q4:"$query4 + if [ "$md5sum" != "" ] + then + query4=$query4", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'" + fi + query4=$query4", fHasDrsFile="$numdrsfiles + query4=$query4", fFitsFileErrors="$numfitserrors + query4=$query4", fRunStart='"$runstart"', fRunStop='"$runstop"'" + query4=$query4" "$querystop + echo "Q4:"$query4 >> $logfile 2>&1 if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query4" then - echo "insert/update of "$numfromfile" to mysql" - fi - exit + echo "insert/update of "$numfromfile" to mysql" >> $logfile 2>&1 + fi + + # print summary for debug + echo "*******" >> $logfile 2>&1 + echo "summary for file "$file >> $logfile 2>&1 + echo "*******" >> $logfile 2>&1 + echo " errors: "$numfitserrors" (1 means errors exist)" >> $logfile 2>&1 + echo " number of drs files: "$numdrsfiles >> $logfile 2>&1 + echo " runnumber error: "$runnumbererror >> $logfile 2>&1 + echo " number error: "$numbererror >> $logfile 2>&1 + echo " roi: "$roi >> $logfile 2>&1 + echo " runtype: "$runtype >> $logfile 2>&1 + echo " numevents: "$numevents >> $logfile 2>&1 + echo " md5sum: "$md5sum >> $logfile 2>&1 + echo " md5sum(zip): "$md5sumzip >> $logfile 2>&1 + echo " start: "$runstart >> $logfile 2>&1 + echo " stop: "$runstop >> $logfile 2>&1 + + # missing + # check wether entry has been made (status table) done done