Index: /trunk/MagicSoft/Mars/Changelog =================================================================== --- /trunk/MagicSoft/Mars/Changelog (revision 8857) +++ /trunk/MagicSoft/Mars/Changelog (revision 8858) @@ -24,4 +24,9 @@ - added check to prevent that checksums of files not read from tape enter the .lapalma file + + * datacenter/scripts/dbchk + - added check for crashed runs, sequences and datasets + - corrected bugs in find command and $dir-loops + - adapted to the automatic concept and added it to the cronjobs Index: /trunk/MagicSoft/Mars/datacenter/scripts/dbchk =================================================================== --- /trunk/MagicSoft/Mars/datacenter/scripts/dbchk (revision 8857) +++ /trunk/MagicSoft/Mars/datacenter/scripts/dbchk (revision 8858) @@ -1,19 +1,38 @@ #!/bin/sh - - -# SETUP - -# directory where the sequence files are stored -sequencedir=/magic/sequences -callistodir=/magic/data/callisto -stardir=/magic/data/star -# access to the sql database -alias mymysql='mysql -s -u MAGIC -h vela --password=d99swMT!' -# database name -db=MyMagic - - +# +# ======================================================================== +# +# * +# * This file is part of MARS, the MAGIC Analysis and Reconstruction +# * Software. It is distributed to you in the hope that it can be a useful +# * and timesaving tool in analysing Data of imaging Cerenkov telescopes. +# * It is distributed WITHOUT ANY WARRANTY. +# * +# * Permission to use, copy, modify and distribute this software and its +# * documentation for any purpose is hereby granted without fee, +# * provided that the above copyright notice appear in all copies and +# * that both that copyright notice and this permission notice appear +# * in supporting documentation. It is provided "as is" without express +# * or implied warranty. +# * +# +# +# Author(s): Thomas Bretz 03/2007 +# Author(s): Daniela Dorner 02/2008 +# Author(s): Stefan Ruegamer 02/2008 +# +# Copyright: MAGIC Software Development, 2000-2008 +# +# +# ======================================================================== +# +# This script searches for crashed runs, sequences or datasets in the db +# as well as obsolete or missing sequence files and corresponding callisto +# and star folders. In the third and fourth check the consistency between +# the Run and Sequence DB is checked. +# +# # TODO - +# # Spped up with SQL: FIELD(25123, 1, 2, 3, 4, 5, ..., 1000200); # Cross check sequences in Sequence and other databases @@ -23,7 +42,48 @@ +source `dirname $0`/sourcefile +printprocesslog "INFO starting $0" +program=dbchk + +scriptlog=$runlogpath/$program-$datetime.log +date >> $scriptlog 2>&1 + +getdbsetup +alias mymysql='mysql -s -u $us --password=$pw --host=vela $db' + +# check for crashed runs, sequences and datasets +echo "Checking if something is crashed on run basis" >> $scriptlog 2>&1 +cruns=`echo "SELECT fRunNumber FROM RunProcessStatus WHERE isnull(fFailedTime) and not isnull(fStartTime) and adddate(fStartTime, Interval 12 HOUR) < Now()" | mymysql` +if [ ! "$cruns" = "" ] +then + printprocesslog "WARN The following runs seem to be crashed. Please check manually: $cruns" + echo "WARN The following runs seem to be crashed. Please check manually: $cruns" >> $scriptlog 2>&1 +else + echo " Nothing found." >> $scriptlog 2>&1 +fi + +echo "Checking if something is crashed on sequence basis" >> $scriptlog 2>&1 +csequences=`echo "SELECT fSequenceFirst FROM SequenceProcessStatus WHERE isnull(fFailedTime) and not isnull(fStartTime) and adddate(fStartTime, Interval 12 HOUR) < Now()" | mymysql` +if [ ! "$csequences" = "" ] +then + printprocesslog "WARN The following sequences seem to be crashed. Please check manually: $csequences" + echo "WARN The following sequences seem to be crashed. Please check manually: $csequences" >> $scriptlog 2>&1 +else + echo " Nothing found." >> $scriptlog 2>&1 +fi + +echo "Checking if something is crashed on dataset basis" >> $scriptlog 2>&1 +cdatasets=`echo "SELECT fDataSetNumber FROM DataSetProcessStatus WHERE isnull(fFailedTime) and not isnull(fStartTime) and adddate(fStartTime, Interval 12 HOUR) < Now()" | mymysql` +if [ ! "$cdatasets" = "" ] +then + printprocesslog "WARN The following datasets seem to be crashed. Please check manually: $cdatasets" + echo "WARN The following datasets seem to be crashed. Please check manually: $cdatasets" >> $scriptlog 2>&1 +else + echo " Nothing found." >> $scriptlog 2>&1 +fi + # CHECK 1 -echo Checking if all sequence files have a corresponding entry in Sequences -files=`find $sequencedir -type f` +echo "Checking if all sequence files have a corresponding entry in Sequences" >> $scriptlog 2>&1 +files=`find $sequpath -type f` for file in $files do @@ -31,12 +91,12 @@ if [ "$sequence" = "" ] then - echo No sequence file: $file + echo "No sequence file: $file" >> $scriptlog 2>&1 continue fi - - var=`echo SELECT fSequenceFirst FROM $db.Sequences WHERE fSequenceFirst=$sequence | mymysql` + + var=`echo SELECT fSequenceFirst FROM Sequences WHERE fSequenceFirst=$sequence | mymysql` if ! [ "$sequence" = "$var" ] then - echo Sequence-File $sequence exist but it is not in Sequences. + echo "Sequence-File $sequence exist but it is not in Sequences." >> $scriptlog 2>&1 continue fi @@ -44,19 +104,19 @@ # CHECK 1b (callisto) -echo Checking if all sequences in $callistodir have a corresponding sequence in Sequence -dirs=`find $callistodir -type d` +echo "Checking if all sequences in ${datapath}/callisto have a corresponding sequence in Sequence" >> $scriptlog 2>&1 +dirs=`find ${datapath}/callisto -mindepth 2 -maxdepth 2 -type d` for dir in $dirs do - sequence=`echo $file | sed -e "s/^.*\/0\+\([0-9]\+\)$/\1/"` + sequence=`echo $dir | sed -e "s/^.*\/0\+\([0-9]\+\)$/\1/"` if [ "$sequence" = "" ] then - echo Invalid directory: $dir + echo "Invalid directory: $dir" >> $scriptlog 2>&1 continue fi - - var=`echo SELECT fSequenceFirst FROM $db.Sequences WHERE fSequenceFirst=$sequence | mymysql` + + var=`echo SELECT fSequenceFirst FROM Sequences WHERE fSequenceFirst=$sequence | mymysql` if ! [ "$sequence" = "$var" ] then - echo $dir exists but no corresponding sequence in Sequences. + echo "$dir exists but no corresponding sequence in Sequences." >> $scriptlog 2>&1 continue fi @@ -64,19 +124,19 @@ # CHECK 1c (star) -echo Checking if all sequences in $stardir have a corresponding sequence in Sequence -dirs=`find $stardir -type d` +echo "Checking if all sequences in ${datapath}/star have a corresponding sequence in Sequence" >> $scriptlog 2>&1 +dirs=`find ${datapath}/star -mindepth 2 -type d` for dir in $dirs do - sequence=`echo $file | sed -e "s/^.*\/0\+\([0-9]\+\)$/\1/"` + sequence=`echo $dir | sed -e "s/^.*\/0\+\([0-9]\+\)$/\1/"` if [ "$sequence" = "" ] then - echo Invalid directory: $dir + echo "Invalid directory: $dir" >> $scriptlog 2>&1 continue fi - - var=`echo SELECT fSequenceFirst FROM $db.Sequences WHERE fSequenceFirst=$sequence | mymysql` + + var=`echo SELECT fSequenceFirst FROM Sequences WHERE fSequenceFirst=$sequence | mymysql` if ! [ "$sequence" = "$var" ] then - echo $dir exists but no corresponding sequence in Sequences. + echo "$dir exists but no corresponding sequence in Sequences." >> $scriptlog 2>&1 continue fi @@ -84,24 +144,24 @@ # CHECK 2 -echo Checking if all sequences in Sequences have a corresponding sequence files -sequences=`echo SELECT fSequenceFirst FROM $db.Sequences | mymysql` +echo "Checking if all sequences in Sequences have a corresponding sequence files" >> $scriptlog 2>&1 +sequences=`echo SELECT fSequenceFirst FROM Sequences | mymysql` for sequence in $sequences do - var=`find $sequencedir -type f -regex .*/sequence0+${sequence}\.txt$` + var=`find $sequpath -type f -regex .*/sequence0+${sequence}\.txt$` if [ "$var" = "" ] then - echo Sequence-File for $sequence not found but in db. + echo "Sequence-File for $sequence not found but in db." >> $scriptlog 2>&1 fi done # CHECK 3 -echo Checking if all sequences from Sequences exists RunData -sequences=`echo SELECT fSequenceFirst FROM $db.Sequences GROUP BY fSequenceFirst | mymysql` +echo "Checking if all sequences from Sequences exists RunData" >> $scriptlog 2>&1 +sequences=`echo SELECT fSequenceFirst FROM Sequences GROUP BY fSequenceFirst | mymysql` for sequence in $sequences do - res=`echo SELECT fSequenceFirst FROM $db.RunData WHERE fSequenceFirst=$sequence GROUP BY fSequenceFirst | mymysql` + res=`echo SELECT fSequenceFirst FROM RunData WHERE fSequenceFirst=$sequence GROUP BY fSequenceFirst | mymysql` if ! [ "$sequence" = "$res" ] then - echo Sequence $sequence exists in Sequences but not in RunData. + echo "Sequence $sequence exists in Sequences but not in RunData." >> $scriptlog 2>&1 continue fi @@ -109,14 +169,15 @@ # CHECK 4 -echo Checking if all sequences from RunData exists in Sequences -sequences=`echo SELECT fSequenceFirst FROM $db.RunData WHERE not fSequenceFirst=0 GROUP BY fSequenceFirst | mymysql` +echo "Checking if all sequences from RunData exists in Sequences" >> $scriptlog 2>&1 +sequences=`echo SELECT fSequenceFirst FROM RunData WHERE not fSequenceFirst=0 GROUP BY fSequenceFirst | mymysql` for sequence in $sequences do - var=`echo SELECT fSequenceFirst FROM $db.Sequences WHERE fSequenceFirst=$sequence | mymysql` + var=`echo SELECT fSequenceFirst FROM Sequences WHERE fSequenceFirst=$sequence | mymysql` if ! [ "$sequence" = "$var" ] then - echo Sequence $sequence exists in RunData but not in Sequences. + echo "Sequence $sequence exists in RunData but not in Sequences." >> $scriptlog 2>&1 continue fi done +finish >> $scriptlog 2>&1