source: trunk/MagicSoft/Mars/datacenter/scripts/jobmanager@ 7933

#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  05/2006 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2006
#
#
# ========================================================================
#
# This a script, which launches other scripts (all scripts, that are run 
# on primary basis
#


source `dirname $0`/sourcefile
user=`whoami`

set -C

echo "" >> $jmscriptlog 2>&1
echo `date`": starting jobmanager" >> $jmscriptlog 2>&1
makedir $jmlogpath
makedir $listpath >> $jmscriptlog 2>&1
makedir $lockpath >> $jmscriptlog 2>&1

notcount=0
while (( $notcount < 100 ))
do
   for (( i=0 ; i < ${#scripts[@]} ; i++ ))
   do 
      makedir $jmlogpath
      date  >> $jmscriptlog 2>&1
      echo "sleeping $sleeptime..." >> $jmscriptlog 2>&1
      sleep $sleeptime
      dayofweek=`date +%u`
      hour=`date +%k`
      source `dirname $0`/sourcefile
      echo "script: ${scripts[$i]}" >> $jmscriptlog 2>&1
#      echo " day: $dayofweek hour: $hour" >> $jmscriptlog 2>&1
      queued=`/usr/local/bin/condor_q -global -format "%s\n" Owner | grep -c $user`
      queued=`setzero $queued`
      queuedscript=`/usr/local/bin/condor_q -global | grep $user | grep -c ${scripts[$i]}`
      queuedscript=`setzero $queuedscript`
      runningscript=`/usr/local/bin/condor_q -global | grep $user | grep ' R ' | grep -c ${scripts[$i]}`
      runningscript=`setzero $runningscript`
      stillinqueue=`echo $queuedscript - $runningscript | bc `

      totalpno=${pnototal[$hour]}
      
      #choose array according to the day of the week
      case $dayofweek in
         0 | 6)  pnos=( ${pnoswe[@]} ) ;;
             *)  pnos=( ${pnosweek[@]} ) ;;
      esac
      num=`echo "((( $i + 1 ) * 24 ) + ( $hour + 1 ) ) - 24 - 1 " | bc `
      pnoscript=${pnos[$num]}
      echo " found $queued jobs in the queue (incl. running jobs) [allowed $totalpno]" >> $jmscriptlog 2>&1
      echo " found $queuedscript ${scripts[$i]} in the queue (incl. running jobs) [allowed $pnoscript]" >> $jmscriptlog 2>&1
#      echo "  => num=$num => pnoscript=$pnoscript (i.e. $pnoscript ${scripts[$i]} allowed)" >> $jmscriptlog 2>&1
      
      if [ "$queued" -gt "$totalpno" ]
      then
         cont >> $jmscriptlog 2>&1
      else
         if [ "$queuedscript" -gt "$pnoscript" ]
         then
            cont >> $jmscriptlog 2>&1
         fi
      fi

      # check if there's something to do
      getstatus
      if [ "$numproc" = "" ]
      then 
         cont >> $jmscriptlog 2>&1
      fi
      if ! [ $numproc -gt 0 ]
      then
         cont >> $jmscriptlog 2>&1
      fi
      if [ $numproc -lt $stillinqueue ]
      then 
         echo "numproc($numproc) -lt stillinqueue($stillinqueue)" >> $jmscriptlog 2>&1
         cont >> $jmscriptlog 2>&1
      fi

      date=`date +%Y/%m/%d`
      date2=`date +%Y-%m-%d`
      condordir=$logpath/condor/$date
      makedir $condordir >> $jmscriptlog 2>&1
      
      if ! /usr/local/bin/condor_submit -a path=$scriptspath -a prog=${scripts[$i]} -a date=$date2 -a dir=$condordir $scriptspath/run.condor 2>> $jmerrorlog
      then 
         date >> $jmerrorlog
         nail -s 'condor problem'  $erradrs < $jmerrorlog
         echo "condor is not working -> sleeping $errorsleeptime" >> $jmscriptlog 2>&1
         sleep $errorsleeptime
      fi
      echo "" >> $jmscriptlog 2>&1
   done
done