Changeset 9129 for trunk/MagicSoft/Mars/datacenter

Timestamp:

08/21/08 18:49:35 (16 years ago)

Author:

Daniela Dorner

Message:

*** empty log message ***

Location:

trunk/MagicSoft/Mars/datacenter/scripts

Files:

• jobmanager (modified) (5 diffs)
• runcamera (modified) (1 diff)
• runcorsika (modified) (4 diffs)
• runreflector (modified) (1 diff)
• setup (modified) (5 diffs)
• sourcefile (modified) (5 diffs)

trunk/MagicSoft/Mars/datacenter/scripts/jobmanager

@@ -37 +37 @@
 echo "starting jobmanager ("`date`")" >> $jmscriptlog 2>&1
 
-if [ "$1" == "" ]
-then
-   echo "running jobmanager for data" >> $jmscriptlog 2>&1
-   scripts=( ${datascripts[@]} )
-   scriptscolname=( ${datascriptscolname[@]} )
-elif [ "$1" = "mc" ]
-then
-   echo "running jobmanager for mc" >> $jmscriptlog 2>&1
-   scripts=( ${mcscripts[@]} )
-   scriptscolname=( ${mcscriptscolname[@]} )
-else
-   echo "$1 is awrong commandline option for jobmanager -> exit" >> $jmscriptlog 2>&1
-   printprocesslog "WARN $1 is wrong commandline option for jobmanager"
-   finish >> $jmscriptlog 2>&1
-fi
+# decide which jobmanager you want to run
+# setup of the different jobmanagers (which scripts they start)
+#  the number of jobs are defined in the file setup
+case $1 in
+   data) echo "running jobmanager for data" >> $jmscriptlog 2>&1
+         scripts=( "runganymed" "runstar" "runcallisto" ) # not used: "dodatacheck" "cutslices"
+         scriptscolname=( "fGanymed" "fStar" "fCallisto" ) # not used: "fDataCheckDone" "fCompmux" 
+         pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ) # not used: ${pnodatacheck[@]} ${pnocutslices[@]}
+         pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ) # not used: ${pnodatacheckwe[@]} ${pnocutsliceswe[@]}
+         break
+         ;;
+     mc) echo "running jobmanager for mc" >> $jmscriptlog 2>&1
+         scripts=( "runcorsika" "runreflector" "runcamera" )
+         scriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
+         pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} )
+         pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )
+         break
+         ;;
+      *) echo "$1 is a wrong commandline option for jobmanager -> exit" >> $jmscriptlog 2>&1
+         printprocesslog "WARN $1 is wrong commandline option for jobmanager"
+         finish >> $jmscriptlog 2>&1
+         break
+         ;;
+esac
+
+# choose commands according to queueing system (defined in setup)
+case $queuesys in
+      sge)  echo "setting commands for sun grid engine" >> $jmscriptlog 2>&1
+            alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -b y -e `echo $runlogpath`/error-`echo $date`.log -o `echo $runlogpath`/log-`echo $date`.log `echo $scriptspath`/`echo ${scripts[$i]}` '
+#            alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -sc runlogpath=`echo $runlogpath` -sc date=`echo $date` -sc scriptspath=`echo $scriptspath` -sc script=`echo ${scripts[$i]}` `echo $scriptspath`/job.sge '
+            alias 'checkqueue'="/opt/gridengine/bin/lx26-amd64/qstat | awk ' { print \"Owner\"\$4\" \" \$3\"Jobstatus\"\$5 } '"
+            break
+            ;;
+   condor)  echo "setting commands for condor" >> $jmscriptlog 2>&1
+            alias 'queuesubmit'='/usr/local/bin/condor_submit -a path=`echo $scriptspath` -a prog=`echo ${scripts[$i]}` -a date=`echo $date` -a dir=`echo $runlogpath` `echo $scriptspath`/run.condor'
+            alias 'checkqueue'='/usr/local/bin/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus '
+            break
+            ;;
+        *)  finish >> $jmscriptlog 2>&1
+            ;;
+esac
 
 prev=$max
@@ -59 +85 @@
 nothingtodocount=0
 nothingtodosleeptime=0
+errorsleeptime=$errorsleeptimedefault
 while (( $notcount < 100 ))
 do
@@ -96 +123 @@
 
       # get processes in queue
-      q=(`/usr/local/bin/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus 2>&1 `)
+#      q=(`/usr/local/bin/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus 2>&1 `)
+#      q=(`/opt/gridengine/bin/lx26-amd64/qstat | awk ' { print "Owner"$4" " $3"Jobstatus"$5 } ' 2>&1 `)
+      q=(`checkqueue 2>&1 `)
       if echo $q | egrep \(Error\|failed\)
       then 
-         echo `date`" WARN condor_q failed" >> $jmscriptlog 2>&1
-         printprocesslog "WARN condor_q failed"
-         echo `date`" WARN condor_q failed" >> $jmerrorlog
+         echo `date`" WARN checking query ($queuesys) failed" >> $jmscriptlog 2>&1
+         printprocesslog "WARN checking query ($queuesys) failed"
+         echo `date`" WARN checking query ($queuesys) failed" >> $jmerrorlog
          cont >> $jmscriptlog 2>&1
       fi
@@ -111 +140 @@
       queuedscript=${#q2[@]}
       # get running scripts
-      q3=(`echo ${q[@]} | egrep -o ${scripts[$i]}Jobstatus2`)
+      q3=( `echo ${q[@]} | egrep -o \(${scripts[$i]}Jobstatus2\|${scripts[$i]}Jobstatusr\)` )
       runningscript=${#q3[@]}
       stillinqueue=`echo $queuedscript - $runningscript | bc `
@@ -151 +180 @@
       prev=$max
 
-      # submit 1 script to condor
+      # submit 1 script to queuing system
       date=`date +%Y-%m-%d`
-      echo " committing 1 ${scripts[$i]} to condor" >> $jmscriptlog 2>&1
-      if ! /usr/local/bin/condor_submit -a path=$scriptspath -a prog=${scripts[$i]} -a date=$date -a dir=$runlogpath $scriptspath/run.condor 2>> $jmerrorlog
-      then 
-         echo `date`" WARN condor_submit failed" >> $jmerrorlog
-         echo "condor is not working -> sleeping $errorsleeptime" >> $jmscriptlog 2>&1
-         printprocesslog "WARN submitting ${scripts[$i]} to condor failed"
+      echo " committing 1 ${scripts[$i]} to $queuesys" >> $jmscriptlog 2>&1
+#      if ! /usr/local/bin/condor_submit -a path=$scriptspath -a prog=${scripts[$i]} -a date=$date -a dir=$runlogpath $scriptspath/run.condor 2>> $jmerrorlog
+#      if ! /opt/gridengine/bin/lx26-amd64/qsub -e $runlogpath/error-$date.log -o $runlogpath/log-$(date).log 2>> $jmerrorlog
+      if ! queuesubmit 2>> $jmerrorlog
+      then 
+         echo `date`" WARN submitting job ($queuesys) failed" >> $jmerrorlog
+         echo "$queuesys is not working -> sleeping $errorsleeptime" >> $jmscriptlog 2>&1
+         printprocesslog "WARN submitting ${scripts[$i]} ($queuesys) failed"
          if [ $errorsleeptime -lt $sleeptimelimit ]
          then 

trunk/MagicSoft/Mars/datacenter/scripts/runcamera

@@ -32 +32 @@
 program=camera
 column=fCameraFileAvail
-detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector
 
 set -C

trunk/MagicSoft/Mars/datacenter/scripts/runcorsika

@@ -35 +35 @@
 set -C
 
-export FLUPRO='/home/operator/Corsika/Mmcs6500/fluka2006_3'
+export FLUPRO=$corsikapath'/fluka2006_3'
 
 # printing input card to stdout
@@ -73 +73 @@
    echo "CERTEL 1"
    echo " 0. 0. 0. 0. 0. 1800. 1700."
-   echo "USER operator"
+   echo "USER "`whoami`
    echo "ATMOSPHERE 11 T"
    echo "EXIT"
@@ -83 +83 @@
 
 # get sequence # 
-gettodo >> $scriptlog 2>&1
-run=$process
+gettodo "1" >> $scriptlog 2>&1
+run=${primaries[0]}
 # get corsika runno
 query="SELECT fCorsikaRunNumber FROM MCRunData where fMCRunNumber="$run
@@ -136 +136 @@
 makedir $outpath >> $scriptlog 2>&1
 
-cd /home/operator/Corsika/Mmcs6500/
+cd $corsikapath
 
 echo "INPUTCARD:" >> $logfile 2>&1

trunk/MagicSoft/Mars/datacenter/scripts/runreflector

@@ -32 +32 @@
 program=reflector
 column=fReflectorFileAvail
-detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector
 
 set -C

trunk/MagicSoft/Mars/datacenter/scripts/setup

@@ -37 +37 @@
 fi
 
+# queuing system
+queusys=condor # wuerzburg data center
+#queuesys=sge  # isdc
 
+# logging and setup
 logpath=/magic/datacenter/autologs
 lockpath=/magic/datacenter/locks
 setuppath=/magic/datacenter/setup
 
+# paths of data
 datapath=/magic/data
 subsystempath=/magic/subsystemdata
@@ -47 +52 @@
 datasetpath=/magic/datasets
 
+# paths and setup for mc
 mcpath=/magic/montecarlo
 mcrawpath=$mcpath/rawfiles
 mcsequpath=$mcpath/sequences
+corsikapath=/home/operator/Corsika/Mmcs6500/
+detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector
 reflectorversion="reflector 0.7"
-cameraversion="camera 0.75"
+cameraversion="camera 0.8"
 
 webpath=/www/htdocs/datacenter
@@ -66 +74 @@
 errorsleeptimedefault=60 #60
 max=42 #maximum number of processes
-
-# not done/used
-#scripts=( "dodatacheck" "cutslices" )
-#scriptscolname=( "fDataCheckDone" "fCompmux" )
-# currently used
-datascripts=( "runganymed" "runstar" "runcallisto" )
-datascriptscolname=( "fGanymed" "fStar" "fCallisto" )
 
 #
@@ -91 +92 @@
 #pnocutslices=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
 #pnodatacheck=(   20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
+pnocorsika=(     12 12 12 12 12 12 12 10  9  8  7  7  7  7  7  7  7  7  7  8  9 10 12 12 )
+pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
+pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
 
 pnocallistowe=(  32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 )
@@ -97 +101 @@
 #pnocutsliceswe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
 #pnodatacheckwe=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
+pnocorsikawe=(   12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 )
+pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
+pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
 
-# not used/done
-#pnosweek=( ${pnodatacheck[@]} ${pnocutslices[@]} )
-#pnoswe=( ${pnodatacheckwe[@]} ${pnocutsliceswe[@]} )
-# currently used
-pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} )
-pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} )
-
-
-# not yet working completely
-
-## values for mc production
-#mcscripts=( "runcorsika" "runreflector" "runcamera" )
-#mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
-#
-##hour:            0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
-#
-#pnocorsika=(     12 12 12 12 12 12 12 10  9  8  7  7  7  7  7  7  7  7  7  8  9 10 12 12 )
-#pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
-#pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
-#
-#pnocorsikawe=(   12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 )
-#pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
-#pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
-#
-#pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} )
-#pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )
-

trunk/MagicSoft/Mars/datacenter/scripts/sourcefile

@@ -231 +231 @@
    pw=`grep Password $mars/sql.rc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
    us=`grep User $mars/sql.rc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
+   ho=`grep URL $mars/sql.rc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
 #   echo "setup: "
 #   echo " db: "$db
 #   echo " pw: "$pw
 #   echo " us: "$us
+#   echo " ho: "$ho
 }
 
@@ -284 +286 @@
    fi
    echo " QUERY: "$query
-   if ! process=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
+   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
    then
       echo "ERROR could not query processes from db -> exit"
@@ -321 +323 @@
    query=$query" group by $column "
 #   echo " QUERY: "$query
-   if ! numproc=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
+   if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
    then
       echo "ERROR could not query number of todo proceses from db -> continue"
@@ -373 +375 @@
    done
    echo " QUERY: "$query
-   if ! mysql -s -u $us --password=$pw --host=vela $db -e " $query "
+   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
    then
       echo "ERROR could not insert status into db -> exit"
@@ -386 +388 @@
 {
    getdbsetup
-   if ! val=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
+   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
    then
       printprocesslog "ERROR could not query db (program: $program, function sendquery)"