Index: trunk/MagicSoft/Mars/Changelog =================================================================== --- trunk/MagicSoft/Mars/Changelog (revision 7911) +++ trunk/MagicSoft/Mars/Changelog (revision 7912) @@ -80,5 +80,6 @@ - implemented usage of gettodo instead of getdolist (faster) - * datacenter/scripts/dodatacheck: + * datacenter/scripts/dodatacheck, datacenter/scripts/runcallisto, + datacenter/scripts/runganymed, datacenter/scripts/runstar: - improved logging and comments - adapted find-to-do-file-algorithm Index: trunk/MagicSoft/Mars/datacenter/scripts/runcallisto =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/runcallisto (revision 7911) +++ trunk/MagicSoft/Mars/datacenter/scripts/runcallisto (revision 7912) @@ -58,47 +58,29 @@ # get todo file -echo "checking if other todo-files are there" >> $scriptlog 2>&1 -if ls $todofile-[1-9]*.txt >> $scriptlog 2>&1 +possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt` >> $scriptlog 2>&1 +if [ "$possibletodofiles" = "" ] then - echo "other file(s) on disk " >> $scriptlog 2>&1 - echo " -> choose one file and start calibrating" >> $scriptlog 2>&1 + echo "ERROR: in $program no todofiles found => something went wrong in jobmanager" + finish >> $scriptlog 2>&1 else - # check if getting of list is already running - checklock "getting list of" >> $scriptlog 2>&1 - # get todo list - gettodo >> $scriptlog 2>&1 - rm -v $lockfile >> $scriptlog 2>&1 + singleprocess="yes" + echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1 + for possibletodofile in ${possibletodofiles[@]} + do + if ! ls $possibletodofile >> $scriptlog 2>&1 + then + echo "file is not on disk -> continue" >> $scriptlog 2>&1 + continue + fi + lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'` + checklock >> $scriptlog 2>&1 + todofile=$possibletodofile + done fi - -# choose todo file -nr=bla -echo "finding the right todo-file" >> $scriptlog 2>&1 -todofiles=`ls -r $listpath/ToDo-$table-$column-*` - -for todofile in ${todofiles[@]} -do - if ! ls $todofile >> $scriptlog 2>&1 - then - echo "file is not on disk -> continue" >> $scriptlog 2>&1 - continue - fi - lockfile=`echo $todofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'` - date > $lockfile >> $scriptlog 2>&1 - checklock=$? - case $checklock in - 0) echo "checklock=$checklock -> setting number" >> $scriptlog 2>&1 - nr=${i} - break;; - 1) echo "checklock=$checklock -> file exists -> continue" >> $scriptlog 2>&1;; - *) echo "checklock=$checklock -> something went completely wrong" >> $scriptlog 2>&1;; - esac -done - -case $nr in - bla) echo "everything is beeing processed -> exit" >> $scriptlog 2>&1 - date >> $scriptlog 2>&1 - exit;; - 12345678) echo "process nr: "$nr >> $scriptlog 2>&1;; -esac +if [ "$todofile" = "" ] +then + echo "no todofile found -> exit" >> $scriptlog 2>&1 + finish >> $scriptlog 2>&1 +fi # get sequence(s) from todo file @@ -116,4 +98,5 @@ for sequence in ${sequences[@]} do + echo "run $program for sequence $sequence..." >> $scriptlog 2>&1 no=`printf %08d $sequence | cut -c 0-4` no2=`printf %08d $sequence` @@ -121,9 +104,7 @@ var2=$no2 outpath="$datapath/$program/$no/$no2" - echo "outpath: "$outpath >> $scriptlog 2>&1 makedir $outpath >> $scriptlog 2>&1 sequfile="$sequpath/$no/sequence$no2.txt" - echo "sequfile: "$sequfile >> $scriptlog 2>&1 # find callisto.rc file @@ -143,5 +124,4 @@ fi - echo "run $program..." >> $scriptlog 2>&1 setstatus "start" >> $scriptlog 2>&1 @@ -150,13 +130,13 @@ case $check1 in - 0) echo "check1=$check1 -> everything ok -> doing update..." >> $scriptlog 2>&1 + 0) echo " check1=$check1 -> everything ok -> doing update..." >> $scriptlog 2>&1 # running merpp update if calibration worked # finding files, which have to be updated - echo "finding files to be updated" >> $scriptlog 2>&1 + echo "finding files to be updated..." >> $scriptlog 2>&1 calfiles=`find $outpath -name *_Y_* ` - echo "files to be updated: "$calfiles >> $scriptlog 2>&1 + echo " files to be updated: "$calfiles >> $scriptlog 2>&1 if [ "$calfiles" = "" ] then - echo "no files found -> continue with next sequence" >> $scriptlog 2>&1 + echo " no files found -> continue with next sequence" >> $scriptlog 2>&1 continue fi @@ -176,6 +156,4 @@ cacofile=`find /magic/subsystemdata/caco/ -name dc_[2][0][0-2][0-9]_[0,1][0-9]_[0-3][0-9]_*${runno}_${source}.txt` # cacofile=`find /magic/subsystemdata/caco/ -name dc_[2][0][0-2][0-9]_[0,1][0-9]_[0-3][0-9]_*${runno}_*.txt` - echo "runno: "$runno >> $scriptlog 2>&1 - echo "ccfile: "$ccfile >> $scriptlog 2>&1 if [ "$ccfile" = "" ] then @@ -186,5 +164,4 @@ break fi - echo "cacofile: "$cacofile >> $scriptlog 2>&1 if [ "$cacofile" = "" ] then @@ -218,6 +195,6 @@ check2=$? case $check2 in - 0) echo "check2=$check2 -> everything ok, merppccupdate worked -> continue" >> $scriptlog 2>&1;; - *) echo "check2=$check2 -> ERROR -> merppccupdate failed" >> $scriptlog 2>&1 + 0) echo " check2=$check2 -> everything ok, merppccupdate worked -> continue" >> $scriptlog 2>&1;; + *) echo " check2=$check2 -> ERROR -> merppccupdate failed" >> $scriptlog 2>&1 com=$Fmerppcc comadd=$runno @@ -228,6 +205,6 @@ check3=$? case $check3 in - 0) echo "check3=$check3 -> everything ok, merppcacoupdate worked -> continue" >> $scriptlog 2>&1;; - *) echo "check3=$check3 -> ERROR -> merppcacoupdate failed" >> $scriptlog 2>&1 + 0) echo " check3=$check3 -> everything ok, merppcacoupdate worked -> continue" >> $scriptlog 2>&1;; + *) echo " check3=$check3 -> ERROR -> merppcacoupdate failed" >> $scriptlog 2>&1 com=$Fmerppcaco comadd=$runno @@ -237,11 +214,10 @@ done ;; - *) echo "check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 + *) echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 com=$Fcallisto check=$check1 ;; esac - # set status - echo "inserting the status for $program for sequence $sequence into the db" >> $scriptlog 2>&1 + setstatus "stop" >> $scriptlog 2>&1 done Index: trunk/MagicSoft/Mars/datacenter/scripts/runganymed =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/runganymed (revision 7911) +++ trunk/MagicSoft/Mars/datacenter/scripts/runganymed (revision 7912) @@ -53,48 +53,30 @@ date >> $scriptlog 2>&1 -# finding todo file -echo "checking if other todo-files are there" >> $scriptlog 2>&1 -if ls $todofile-[1-9]*.txt >> $scriptlog 2>&1 +# get todo file +possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt` >> $scriptlog 2>&1 +if [ "$possibletodofiles" = "" ] then - echo "other file(s) on disk " >> $scriptlog 2>&1 - echo " -> choose one file and run $program" >> $scriptlog 2>&1 + echo "ERROR: in $program no todofiles found => something went wrong in jobmanager" + finish >> $scriptlog 2>&1 else - # check if getting of list is already running - checklock "getting list of" >> $scriptlog 2>&1 - # get todo list - gettodo >> $scriptlog 2>&1 - rm -v $lockfile >> $scriptlog 2>&1 + singleprocess="yes" + echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1 + for possibletodofile in ${possibletodofiles[@]} + do + if ! ls $possibletodofile >> $scriptlog 2>&1 + then + echo "file is not on disk -> continue" >> $scriptlog 2>&1 + continue + fi + lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'` + checklock >> $scriptlog 2>&1 + todofile=$possibletodofile + done fi - -# choosing todo file -nr=bla -echo "finding the right todo-file" >> $scriptlog 2>&1 -todofiles=`ls $listpath/ToDo-$table-$column-*` - -for todofile in ${todofiles[@]} -do - if ! ls $todofile >> $scriptlog 2>&1 - then - echo "file is not on disk -> continue" >> $scriptlog 2>&1 - continue - fi - lockfile=`echo $todofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'` - date > $lockfile >> $scriptlog 2>&1 - checklock=$? - case $checklock in - 0) echo "checklock=$checklock -> setting number" >> $scriptlog 2>&1 - nr=${i} - break;; - 1) echo "checklock=$checklock -> file exists -> continue" >> $scriptlog 2>&1;; - *) echo "checklock=$checklock -> something went completely wrong" >> $scriptlog 2>&1;; - esac -done - -case $nr in - bla) echo "everything is beeing processed -> exit" >> $scriptlog 2>&1 - date >> $scriptlog 2>&1 - exit;; - 12345678) echo "process nr: "$nr >> $scriptlog 2>&1;; -esac +if [ "$todofile" = "" ] +then + echo "no todofile found -> exit" >> $scriptlog 2>&1 + finish >> $scriptlog 2>&1 +fi # retrieve dataset from todo file @@ -110,4 +92,5 @@ for dataset in ${datasets[@]} do + echo "run $program for dataset $dataset..." >> $scriptlog 2>&1 no=`printf %08d $dataset | cut -c 0-5` no2=`printf %08d $dataset` @@ -115,9 +98,7 @@ var2=$no2 outpath="$datapath/$program/$no/$no2" - echo "outpath: "$outpath >> $scriptlog 2>&1 makedir $outpath >> $scriptlog 2>&1 datasetfile="$datasetpath/$no/dataset$no2.txt" - echo "datasetfile: "$datasetfile >> $scriptlog 2>&1 # get observation mode to choose ganymed.rc file wobble=`grep 'WobbleMode:' $datasetfile` >> $scriptlog 2>&1 @@ -131,5 +112,4 @@ ganymedrc=$setuppath/ganymed/ganymed_$mode.rc - echo "run $program..." >> $scriptlog 2>&1 setstatus "start" >> $scriptlog 2>&1 @@ -138,11 +118,11 @@ case $check1 in - 0) echo "check1=$check1 -> everything ok " >> $scriptlog 2>&1 ;; - *) echo "check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 + 0) echo " check1=$check1 -> everything ok " >> $scriptlog 2>&1 ;; + *) echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 com=$Fganymed check=$check1 ;; esac - echo "inserting the status for $program for dataset $dataset into the db" >> $scriptlog 2>&1 + setstatus "stop" >> $scriptlog 2>&1 done Index: trunk/MagicSoft/Mars/datacenter/scripts/runstar =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/runstar (revision 7911) +++ trunk/MagicSoft/Mars/datacenter/scripts/runstar (revision 7912) @@ -53,48 +53,30 @@ date >> $scriptlog 2>&1 -# finding todo file -echo "checking if other todo-files are there" >> $scriptlog 2>&1 -if ls $todofile-[1-9]*.txt >> $scriptlog 2>&1 +# get todo file +possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt` >> $scriptlog 2>&1 +if [ "$possibletodofiles" = "" ] then - echo "other file(s) on disk " >> $scriptlog 2>&1 - echo " -> choose one file and start calculation of image parameter" >> $scriptlog 2>&1 + echo "ERROR: in $program no todofiles found => something went wrong in jobmanager" + finish >> $scriptlog 2>&1 else - # check if getting of list is already running - checklock "getting list of" >> $scriptlog 2>&1 - # get todo list - gettodo >> $scriptlog 2>&1 - rm -v $lockfile >> $scriptlog 2>&1 + singleprocess="yes" + echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1 + for possibletodofile in ${possibletodofiles[@]} + do + if ! ls $possibletodofile >> $scriptlog 2>&1 + then + echo "file is not on disk -> continue" >> $scriptlog 2>&1 + continue + fi + lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'` + checklock >> $scriptlog 2>&1 + todofile=$possibletodofile + done fi - -# choosing todo file -nr=bla -echo "finding the right todo-file" >> $scriptlog 2>&1 -todofiles=`ls -r $listpath/ToDo-$table-$column-*` - -for todofile in ${todofiles[@]} -do - if ! ls $todofile >> $scriptlog 2>&1 - then - echo "file is not on disk -> continue" >> $scriptlog 2>&1 - continue - fi - lockfile=`echo $todofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'` - date > $lockfile >> $scriptlog 2>&1 - checklock=$? - case $checklock in - 0) echo "checklock=$checklock -> setting number" >> $scriptlog 2>&1 - nr=${i} - break;; - 1) echo "checklock=$checklock -> file exists -> continue" >> $scriptlog 2>&1;; - *) echo "checklock=$checklock -> something went completely wrong" >> $scriptlog 2>&1;; - esac -done - -case $nr in - bla) echo "everything is beeing processed -> exit" >> $scriptlog 2>&1 - date >> $scriptlog 2>&1 - exit;; - 12345678) echo "process nr: "$nr >> $scriptlog 2>&1;; -esac +if [ "$todofile" = "" ] +then + echo "no todofile found -> exit" >> $scriptlog 2>&1 + finish >> $scriptlog 2>&1 +fi # retrieve sequence from todo file @@ -112,4 +94,5 @@ for sequence in ${sequences[@]} do + echo "run $program for sequence $sequence..." >> $scriptlog 2>&1 no=`printf %08d $sequence | cut -c 0-4` no2=`printf %08d $sequence` @@ -118,12 +101,8 @@ outpath="$datapath/$program/$no/$no2" inpath=`echo $outpath | sed -e 's/star/callisto/'` - echo "inpath: "$inpath >> $scriptlog 2>&1 - echo "outpath: "$outpath >> $scriptlog 2>&1 makedir $outpath >> $scriptlog 2>&1 sequfile="$sequpath/$no/sequence$no2.txt" - echo "sequfile: "$sequfile >> $scriptlog 2>&1 - echo "run $program..." >> $scriptlog 2>&1 setstatus "start" >> $scriptlog 2>&1 @@ -132,11 +111,11 @@ case $check1 in - 0) echo "check1=$check1 -> everything ok " >> $scriptlog 2>&1;; - *) echo "check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 + 0) echo " check1=$check1 -> everything ok " >> $scriptlog 2>&1;; + *) echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 com=$Fstar check=$check1 ;; esac - echo "inserting the status for $program for sequence $sequence into the db" >> $scriptlog 2>&1 + setstatus "stop" >> $scriptlog 2>&1 done