#!/bin/sh # # ======================================================================== # # * # * This file is part of MARS, the MAGIC Analysis and Reconstruction # * Software. It is distributed to you in the hope that it can be a useful # * and timesaving tool in analysing Data of imaging Cerenkov telescopes. # * It is distributed WITHOUT ANY WARRANTY. # * # * Permission to use, copy, modify and distribute this software and its # * documentation for any purpose is hereby granted without fee, # * provided that the above copyright notice appear in all copies and # * that both that copyright notice and this permission notice appear # * in supporting documentation. It is provided "as is" without express # * or implied warranty. # * # # # Author(s): Daniela Dorner 05/2006 # # Copyright: MAGIC Software Development, 2000-2010 # # # ======================================================================== # # This a script, which launches other scripts (all scripts, that are run # on primary basis) # source `dirname $0`/sourcefile printprocesslog "INFO starting $0" set -C # function to continue in loop and go to next script function nextscript() { echo `date +%F\ %T`" sleeping \$$1 = $2 seconds... " >> $jmscriptlog 2>&1 sleep $2 echo "" >> $jmscriptlog 2>&1 continue } echo "" >> $jmscriptlog 2>&1 echo "" >> $jmscriptlog 2>&1 echo -n `date +%F\ %T`" starting jobmanager for setup "$AUTOMATIONSETUP >> $jmscriptlog 2>&1 # choose commands according to queueing system (defined in setup) case $queuesys in sge) echo " on queuing system 'sun grid engine'" >> $jmscriptlog 2>&1 # (-hard) -l hostname=compute-* # for qstat this returns the jobs running on that node + all jobs in the queue alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -b y -v AUTOMATIONSETUP=$AUTOMATIONSETUP -e `echo $runlogpath`/error-`echo $date`.log -o `echo $runlogpath`/log-`echo $date`.log `echo $noderequirementsub` `echo $scriptspath`/`echo ${scripts[$i]}` ' # alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -b y -v AUTOMATIONSETUP=$AUTOMATIONSETUP -e `echo $runlogpath`/error-`echo $date`.log -o `echo $runlogpath`/log-`echo $date`.log `echo $scriptspath`/`echo ${scripts[$i]}` ' # alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -sc runlogpath=`echo $runlogpath` -sc date=`echo $date` -sc scriptspath=`echo $scriptspath` -sc script=`echo ${scripts[$i]}` `echo $scriptspath`/job.sge ' # FIXME: get complete scriptname (including command line option), needed for runstereo alias 'checkqueue'="/opt/gridengine/bin/lx26-amd64/qstat \`echo \$noderequirementstat\` | awk ' { print \"Owner\"\$4\" \" \$3\"Jobstatus\"\$5 } '" break ;; pbs) echo " on queuing system 'pbs'" >> $jmscriptlog 2>&1 alias 'queuesubmit'='$pbspath/qsub -l walltime=$walltime -l pmem=$pmem -v AUTOMATIONSETUP=$AUTOMATIONSETUP,SOURCEFILEPATH=$SOURCEFILEPATH -e `echo $runlogpath`/error-`echo $date`.log -o `echo $runlogpath`/log-`echo $date`.log `echo $noderequirementsub` `echo $scriptspath`/`echo ${scripts[$i]}` ' alias 'checkqueue'="$pbspath/qstat -a | awk ' { print \"Owner\"\$2\" \" \$4\"Jobstatus\"\$10 } '" break ;; condor) echo " on queuing system 'condor'" >> $jmscriptlog 2>&1 alias 'queuesubmit'='$condorpath/condor_submit -a path=`echo $scriptspath` -a prog=`echo ${scripts[$i]}` -a date=`echo $date` -a dir=`echo $runlogpath` -a automationsetup=$AUTOMATIONSETUP `echo $scriptspath`/run.condor' alias 'checkqueue'='$condorpath/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus ' break ;; *) echo "" >> $jmscriptlog 2>&1 finish >> $jmscriptlog 2>&1 ;; esac echo "" >> $jmscriptlog 2>&1 prev=$max user=`whoami` currentnode=$minnode numevaluated=0 # endless loop notcount=0 nothingtodocount=0 nothingtodosleeptime=0 errorsleeptime=$errorsleeptimedefault while (( $notcount < 100 )) do for (( i=0 ; i < ${#scripts[@]} ; i++ )) do source `dirname $0`/sourcefile echo `date +%F\ %T`" Evaluating processing status for script '${scripts[$i]}'" >> $jmscriptlog 2>&1 # check if there's something to do step=${scriptscolname[$i]} getstepinfo # check if walltime has to be set if [ "$setwalltime" = "yes" ] then walltime=${walltimes[$i]} fi # check if memory has to be set if [ "$setpmem" = "yes" ] then pmem=${pmems[$i]} fi # check if the script is restricted to one node # (i.e. where output of previous step(s) is stored) # this information is taken from the steps.rc file if [ "$noderestricted" = "yes" ] then # get number of next node if [ $numevaluated -ge $numrestrictedscripts ] then currentnode=`echo $currentnode + 1 | bc -l` numevaluated=1 else numevaluated=`echo $numevaluated + 1 | bc -l` fi if [ $currentnode -gt $maxnode ] then currentnode=$minnode fi # check if node is excluded for excludednode in ${excludednodes[@]} do if [ $currentnode -eq $excludednode ] then echo `date +%F\ %T`" Node compute-0-$currentnode is currently excluded." >> $jmscriptlog 2>&1 continue 2 fi done # define requirement for submission # FIXME: currently only for sge at isdc echo `date +%F\ %T`" Checking for node $currentnode. " >> $jmscriptlog 2>&1 noderequirementsub=" -hard -l hostname=compute-0-${currentnode}" noderequirementstat=" -l hostname=compute-0-${currentnode}" getstatus $currentnode >> $jmscriptlog 2>&1 else noderequirementsub="" noderequirementstat="" getstatus >> $jmscriptlog 2>&1 fi # check number of processes to be done echo `date +%F\ %T`" Database: $numproc ${scripts[$i]} still to be done (incl. idle jobs) [DB/step $db/$step]" >> $jmscriptlog 2>&1 if [ "$numproc" = "0" ] then prev=0 nothingtodocount=`expr $nothingtodocount + 1` if [ $nothingtodocount -lt ${#scripts[@]} ] then nextscript 0 0 else if [ $nothingtodosleeptime -lt $sleeptimelimit ] then nothingtodosleeptime=`echo " $nothingtodocount * $sleeptime " | bc` fi nextscript nothingtodosleeptime $nothingtodosleeptime fi else nothingtodocount=0 nothingtodosleeptime=0 fi # get processes in queue q=(`checkqueue 2>&1 `) if echo $q | egrep \(Error\|failed\) then echo `date +%F\ %T`" WARN checking query ($queuesys) failed" >> $jmscriptlog 2>&1 printprocesslog "WARN checking query ($queuesys) failed" echo `date +%F\ %T`" WARN checking query ($queuesys) failed" >> $jmerrorlog nextscript sleeptime $sleeptime fi # FIXME: sge cuts scriptname to 8 digits in qstat # get processes of user in queue q1=( `echo ${q[@]} | egrep -o "Owner$user"`) queued=${#q1[@]} # get scripts in queue q2=( `echo ${q[@]} | egrep -o "${scripts[$i]}"`) queuedscript=${#q2[@]} # get running scripts # condor: 2 # sge: r # pbs: R q3=( `echo ${q[@]} | egrep -o \("${scripts[$i]}"Jobstatus2\|"${scripts[$i]}"Jobstatusr\|"${scripts[$i]}"JobstatusR\)` ) runningscript=${#q3[@]} stillinqueue=`echo $queuedscript - $runningscript | bc ` #get total number of allowed process for current time hour=`date +%k` if [ ${pnototal[$hour]} -lt $totalmax ] then totalpno=${pnototal[$hour]} else totalpno=$totalmax fi #choose array according to the day of the week dayofweek=`date +%u` case $dayofweek in 0 | 6) pnos=( ${pnoswe[@]} ) ;; *) pnos=( ${pnosweek[@]} ) ;; esac # get number of allowed scripts for current time num=`echo "((( $i + 1 ) * 24 ) + ( $hour + 1 ) ) - 24 - 1 " | bc ` pnoscript=${pnos[$num]} # if there was nothing to do for previous script, more scripts can be allowed if [ $prev -eq 0 ] then echo `date +%F\ %T`" \$prev=0 => resetting \$pnoscript from $pnoscript to $max [\$max]" >> $jmscriptlog 2>&1 pnoscript=$max fi echo `date +%F\ %T`" queue for user '$user': total: $queued queued jobs [allowed \$totalpno = $totalpno]" >> $jmscriptlog 2>&1 echo `date +%F\ %T`" queue for user '$user': ${scripts[$i]}: $queuedscript queued, $runningscript running, $stillinqueue idle [allowed \$pnoscript = $pnoscript]" >> $jmscriptlog 2>&1 # continue if there are already enough processes or scripts in the queue if [ "$queued" -ge "$totalpno" ] || [ "$queuedscript" -ge "$pnoscript" ] then nextscript sleeptime $sleeptime fi # continue if the number of script is the queue is larger (or equal) than the number which still has to be done if [ $numproc -le $stillinqueue ] then echo `date +%F\ %T`" \$numproc ($numproc) < \$stillinqueue ($stillinqueue) " >> $jmscriptlog 2>&1 nextscript sleeptime $sleeptime fi # reset prev prev=$max # submit 1 script to queuing system date=`date +%Y-%m-%d` echo `date +%F\ %T`" committing 1 ${scripts[$i]} to $queuesys" >> $jmscriptlog 2>&1 if ! queuesubmit 2>> $jmerrorlog then echo `date`" WARN submitting job ($queuesys) failed" >> $jmerrorlog echo `date +%F\ %T`" WARN $queuesys is not working -> sleeping $errorsleeptime [\$errorsleeptime]" >> $jmscriptlog 2>&1 printprocesslog "WARN submitting ${scripts[$i]} ($queuesys) failed" if [ $errorsleeptime -lt $sleeptimelimit ] then errorsleeptime=`echo " $errorsleeptime + $errorsleeptimedefault " | bc` fi nextscript errorsleeptime $errorsleeptime else errorsleeptime=$errorsleeptimedefault fi nextscript sleeptime $sleeptime done done