Index: trunk/MagicSoft/Mars/datacenter/scripts/jobmanager =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/jobmanager (revision 9488) +++ trunk/MagicSoft/Mars/datacenter/scripts/jobmanager (revision 9489) @@ -20,5 +20,5 @@ # Author(s): Daniela Dorner 05/2006 # -# Copyright: MAGIC Software Development, 2000-2007 +# Copyright: MAGIC Software Development, 2000-2009 # # @@ -45,39 +45,5 @@ echo "" >> $jmscriptlog 2>&1 echo "" >> $jmscriptlog 2>&1 -echo -n `date +%F\ %T`" starting jobmanager for setup " >> $jmscriptlog 2>&1 - -# decide which jobmanager you want to run -# setup of the different jobmanagers (which scripts they start) -# the number of jobs are defined in the file setup -case $1 in - data) echo -n "'data'" >> $jmscriptlog 2>&1 - scripts=( "runganymed" "runstar" "runcallisto" ) # not used: "dodatacheck" "cutslices" - scriptscolname=( "fGanymed" "fStar" "fCallisto" ) # not used: "fDataCheckDone" "fCompmux" - pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ) # not used: ${pnodatacheck[@]} ${pnocutslices[@]} - pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ) # not used: ${pnodatacheckwe[@]} ${pnocutsliceswe[@]} - break - ;; - mc) echo -n "'mc'" >> $jmscriptlog 2>&1 - scripts=( "runcorsika" "runreflector" "runcamera" ) - scriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" ) - pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} ) - pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} ) - break - ;; - ctamc) echo -n "'cta mc'" >> $jmscriptlog 2>&1 - scripts=( "runsimtel" ) - scriptscolname=( "fCorsikaSimTelarray" ) - pnosweek=( ${pnosimtel[@]} ) - pnoswe=( ${pnosimtelwe[@]} ) - break - ;; - *) echo "" - echo "'$1' is a wrong commandline option for jobmanager -> exit" >> $jmscriptlog 2>&1 - echo "'$1' is a wrong commandline option for jobmanager -> exit" - printprocesslog "WARN '$1' is a wrong commandline option for jobmanager" - finish >> $jmscriptlog 2>&1 - break - ;; -esac +echo -n `date +%F\ %T`" starting jobmanager for setup "$AUTOMATIONSETUP >> $jmscriptlog 2>&1 # choose commands according to queueing system (defined in setup) Index: trunk/MagicSoft/Mars/datacenter/scripts/setup.isdc.cta =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/setup.isdc.cta (revision 9489) +++ trunk/MagicSoft/Mars/datacenter/scripts/setup.isdc.cta (revision 9489) @@ -0,0 +1,116 @@ +#!/bin/sh +# +# ======================================================================== +# +# * +# * This file is part of MARS, the MAGIC Analysis and Reconstruction +# * Software. It is distributed to you in the hope that it can be a useful +# * and timesaving tool in analysing Data of imaging Cerenkov telescopes. +# * It is distributed WITHOUT ANY WARRANTY. +# * +# * Permission to use, copy, modify and distribute this software and its +# * documentation for any purpose is hereby granted without fee, +# * provided that the above copyright notice appear in all copies and +# * that both that copyright notice and this permission notice appear +# * in supporting documentation. It is provided "as is" without express +# * or implied warranty. +# * +# +# +# Author(s): Daniela Dorner 08/2009 +# +# Copyright: MAGIC Software Development, 2000-2009 +# +# +# ======================================================================== +# +# This a resource file for the scripts, in which the standard paths and +# functions, which are needed more often are stored. +# + +if [ "$rootsys" = "" ] +then + rootsys=/opt/root5.20.x86_64 +fi +if ! export | grep ROOTSYS | grep $rootsys >|/dev/null +then + export ROOTSYS=$rootsys + export PATH=$PATH:$ROOTSYS/bin + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ROOTSYS/lib/root +fi + +# queuing system +queuesys=sge #isdc + +# logging and setup +logpath=/home/dorner/automation/autologs +lockpath=/home/dorner/automation/locks +setuppath=/home/dorner/automation/setup + +## paths of data +#datapath=/magic/data +#subsystempath=/magic/subsystemdata +#sequpath=/magic/sequences +#datasetpath=/magic/datasets + +# paths and setup for mc +mcpath=/state/cta +#mcrawpath=$mcpath/rawfiles +#mcsequpath=$mcpath/sequences +#corsikapath=/home/operator/Corsika/Mmcs6500/ +#detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector +#reflectorversion="reflector 0.7" +#cameraversion="camera 0.8" +#export FLUPRO=$corsikapath'/fluka2008_3' + + +#webpath=/www/htdocs/datacenter + + +# get paths for mars, macros and scripts +mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'` +macrospath=$mars/datacenter/macros +scriptspath=$mars/datacenter/scripts +# rcfiles +# dependencies of steps +steps=$mars/resources/steps_cta_new.rc +# file with db information +sqlrc=$mars/sql.rc + + +#addresses to which the errors are sent +erradrs="daniela.dorner@unige.ch" +#addresses to which the changes are sent +adrs="daniela.dorner@unige.ch" + + +#setup for jobmanager +sleeptime=30 #30 +sleeptimelimit=360 #360 +errorsleeptimedefault=60 #60 +max=500 #maximum number of processes +max=25 #maximum number of processes for one script in case there are more than one and the others do not have anything to do +totalmax=1600 #maximum number of processes (total) overwrites pnototal(we) in case it is smaller + + +# +# FIXME: +# max anz an prozessen setzen +# besser Verhaeltnis zwischen den Prozessen und das am besten noch +# automatisch anpassen +# evtl noch maximale anz an callistos setzen +# + +#hour: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 +pnototal=( 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 ) +pnototalwe=( 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 1500 ) + +pnosimtel=( 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 ) +pnosimtelwe=( 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 65 ) + +# set variables for jobmanager +scripts=( "runsimtel" ) +scriptscolname=( "fCorsikaSimTelarray" ) +pnosweek=( ${pnosimtel[@]} ) +pnoswe=( ${pnosimtelwe[@]} ) + Index: trunk/MagicSoft/Mars/datacenter/scripts/setup.wue.magic.data =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/setup.wue.magic.data (revision 9489) +++ trunk/MagicSoft/Mars/datacenter/scripts/setup.wue.magic.data (revision 9489) @@ -0,0 +1,119 @@ +#!/bin/sh +# +# ======================================================================== +# +# * +# * This file is part of MARS, the MAGIC Analysis and Reconstruction +# * Software. It is distributed to you in the hope that it can be a useful +# * and timesaving tool in analysing Data of imaging Cerenkov telescopes. +# * It is distributed WITHOUT ANY WARRANTY. +# * +# * Permission to use, copy, modify and distribute this software and its +# * documentation for any purpose is hereby granted without fee, +# * provided that the above copyright notice appear in all copies and +# * that both that copyright notice and this permission notice appear +# * in supporting documentation. It is provided "as is" without express +# * or implied warranty. +# * +# +# +# Author(s): Daniela Dorner 08/2009 +# +# Copyright: MAGIC Software Development, 2000-2009 +# +# +# ======================================================================== +# +# This a resource file for the scripts, in which the standard paths and +# functions, which are needed more often are stored. +# + +rootsys=/opt/root_v5.12.00f +if ! export | grep $rootsys >|/dev/null +then + export ROOTSYS=$rootsys + export PATH=$PATH:$ROOTSYS/bin + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ROOTSYS/lib +fi + +# queuing system +queuesys=condor # wuerzburg data center + +# logging and setup +logpath=/magic/datacenter/autologs +lockpath=/magic/datacenter/locks +setuppath=/magic/datacenter/setup + +# paths of data +datapath=/magic/data +subsystempath=/magic/subsystemdata +sequpath=/magic/sequences +datasetpath=/magic/datasets + +## paths and setup for mc +#mcpath=/magic/montecarlo +#mcrawpath=$mcpath/rawfiles +#mcsequpath=$mcpath/sequences +#corsikapath=/home/operator/Corsika/Mmcs6500/ +#detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector +#reflectorversion="reflector 0.7" +#cameraversion="camera 0.8" +#export FLUPRO=$corsikapath'/fluka2008_3' + + +webpath=/www/htdocs/datacenter + +# get paths for mars, macros and scripts +mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'` +macrospath=$mars/datacenter/macros +scriptspath=$mars/datacenter/scripts +# rcfiles +# dependencies of steps +steps=$mars/resources/steps.rc +# file with db information +sqlrc=$mars/sql.rc + + +#addresses to which the errors are sent +erradrs="datacenter@astro.uni-wuerzburg.de" +#addresses to which the changes are sent +adrs="datacenter@astro.uni-wuerzburg.de, tbretz@astro.uni-wuerzburg.de" + + +#setup for jobmanager +sleeptime=30 #30 +sleeptimelimit=360 #360 +errorsleeptimedefault=60 #60 +max=42 #maximum number of processes for one script in case there are more than one and the others do not have anything to do +totalmax=40 #maximum number of processes (total) overwrites pnototal(we) in case it is smaller + +# +# FIXME: +# max anz an prozessen setzen +# besser Verhaeltnis zwischen den Prozessen und das am besten noch +# automatisch anpassen +# evtl noch maximale anz an callistos setzen +# + +#hour: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 +pnototal=( 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 ) +pnototalwe=( 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 ) + +pnocallisto=( 32 32 32 32 32 20 20 18 18 12 12 12 12 12 12 12 12 12 12 18 18 20 20 32 ) +pnostar=( 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 ) +pnoganymed=( 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ) +#pnocutslices=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) +#pnodatacheck=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 ) + +pnocallistowe=( 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 ) +pnostarwe=( 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 ) +pnoganymedwe=( 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ) +#pnocutsliceswe=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) +#pnodatacheckwe=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 ) + +# set variables for jobmanager +scripts=( "runganymed" "runstar" "runcallisto" ) # not used: "dodatacheck" "cutslices" +scriptscolname=( "fGanymed" "fStar" "fCallisto" ) # not used: "fDataCheckDone" "fCompmux" +pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ) # not used: ${pnodatacheck[@]} ${pnocutslices[@]} +pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ) # not used: ${pnodatacheckwe[@]} ${pnocutsliceswe[@]} + Index: trunk/MagicSoft/Mars/datacenter/scripts/setup.wue.magic.mc =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/setup.wue.magic.mc (revision 9489) +++ trunk/MagicSoft/Mars/datacenter/scripts/setup.wue.magic.mc (revision 9489) @@ -0,0 +1,116 @@ +#!/bin/sh +# +# ======================================================================== +# +# * +# * This file is part of MARS, the MAGIC Analysis and Reconstruction +# * Software. It is distributed to you in the hope that it can be a useful +# * and timesaving tool in analysing Data of imaging Cerenkov telescopes. +# * It is distributed WITHOUT ANY WARRANTY. +# * +# * Permission to use, copy, modify and distribute this software and its +# * documentation for any purpose is hereby granted without fee, +# * provided that the above copyright notice appear in all copies and +# * that both that copyright notice and this permission notice appear +# * in supporting documentation. It is provided "as is" without express +# * or implied warranty. +# * +# +# +# Author(s): Daniela Dorner 08/2009 +# +# Copyright: MAGIC Software Development, 2000-2009 +# +# +# ======================================================================== +# +# This a resource file for the scripts, in which the standard paths and +# functions, which are needed more often are stored. +# + +rootsys=/opt/root_v5.12.00f +if ! export | grep $rootsys >|/dev/null +then + export ROOTSYS=$rootsys + export PATH=$PATH:$ROOTSYS/bin + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ROOTSYS/lib +fi + +# queuing system +queuesys=condor # wuerzburg data center + +# logging and setup +logpath=/magic/datacenter/autologs +lockpath=/magic/datacenter/locks +setuppath=/magic/datacenter/setup + +## paths of data +#datapath=/magic/data +#subsystempath=/magic/subsystemdata +#sequpath=/magic/sequences +#datasetpath=/magic/datasets + +# paths and setup for mc +mcpath=/magic/montecarlo +mcrawpath=$mcpath/rawfiles +mcsequpath=$mcpath/sequences +corsikapath=/home/operator/Corsika/Mmcs6500/ +detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector +reflectorversion="reflector 0.7" +cameraversion="camera 0.8" +export FLUPRO=$corsikapath'/fluka2008_3' + + +#webpath=/www/htdocs/datacenter + + +# get paths for mars, macros and scripts +mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'` +macrospath=$mars/datacenter/macros +scriptspath=$mars/datacenter/scripts +# rcfiles +# dependencies of steps +steps=$mars/resources/steps.rc +# file with db information +sqlrc=$mars/sql.rc + + +#addresses to which the errors are sent +erradrs="datacenter@astro.uni-wuerzburg.de" +#addresses to which the changes are sent +adrs="datacenter@astro.uni-wuerzburg.de, tbretz@astro.uni-wuerzburg.de" + + +#setup for jobmanager +sleeptime=30 #30 +sleeptimelimit=360 #360 +errorsleeptimedefault=60 #60 +max=42 #maximum number of processes for one script in case there are more than one and the others do not have anything to do +totalmax=40 #maximum number of processes (total) overwrites pnototal(we) in case it is smaller + +# +# FIXME: +# max anz an prozessen setzen +# besser Verhaeltnis zwischen den Prozessen und das am besten noch +# automatisch anpassen +# evtl noch maximale anz an callistos setzen +# + +#hour: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 +pnototal=( 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 ) +pnototalwe=( 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 ) + +pnocorsika=( 12 12 12 12 12 12 12 10 9 8 7 7 7 7 7 7 7 7 7 8 9 10 12 12 ) +pnoreflector=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) +pnocamera=( 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ) + +pnocorsikawe=( 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 ) +pnoreflectorwe=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) +pnocamerawe=( 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ) + +# set variables for jobmanager +scripts=( "runcorsika" "runreflector" "runcamera" ) +scriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" ) +pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} ) +pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} ) + Index: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/sourcefile (revision 9488) +++ trunk/MagicSoft/Mars/datacenter/scripts/sourcefile (revision 9489) @@ -38,9 +38,11 @@ fi -source `dirname $0`/setup - -mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'` -macrospath=$mars/datacenter/macros -scriptspath=$mars/datacenter/scripts +if [ "$AUTOMATIONSETUP" = "" ] +then + echo "Please set the environment variable \$AUTOMATIONSETUP." + exit +fi + +source `dirname $0`/setup.$AUTOMATIONSETUP datetime=`date +%F-%H-%M-%S` @@ -157,5 +159,6 @@ jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log jmscriptlog=$runlogpath/jobmanager`date +%F`.log -steps=$mars/resources/steps.rc + +# check if rc-files are available if ! ls $steps >/dev/null then @@ -163,5 +166,4 @@ finish fi -sqlrc=$mars/sql.rc if ! ls $sqlrc >/dev/null then @@ -169,4 +171,5 @@ finish fi + # resetting values pno=0 @@ -186,12 +189,4 @@ # in the following the functions, which are needed by several scripts, are # defined - -# function to do continue in a loop and produce according logging -function cont() -{ - date - echo "" - continue -} # function to check if a process is already locked @@ -368,5 +363,6 @@ then printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)" - cont + echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)" + continue fi }