Changeset 9129 for trunk


Ignore:
Timestamp:
08/21/08 18:49:35 (16 years ago)
Author:
Daniela Dorner
Message:
*** empty log message ***
Location:
trunk/MagicSoft/Mars
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/MagicSoft/Mars/Changelog

    r9128 r9129  
    2929   * datacenter/scripts/dowebplots:
    3030     - fixed sequencepath in rsync
     31
     32   * datacenter/scripts/jobmanager:
     33     - changed choosing of jobmanager (data or mc) from if to case
     34     - implemented case-statement for the choice of the queueing
     35       system (condor and sun grid engine implemented up to now)
     36     - implemented commands for sun grid engine
     37
     38   * datacenter/scripts/jobmanager, datacenter/scripts/setup:
     39     - moved defining of array scripts and pnoswe(ek) from sourcefile
     40       to jobmanager
     41     - fixed problem with mc jobmanager pnos
     42
     43   * datacenter/scripts/setup:
     44     - implemented definition of queueing system
     45     - exchanged camera version
     46
     47   * datacenter/scripts/sourcefile:
     48     - host is retrieved from sql.rc now
     49
     50   * datacenter/scripts/runcorsika, datacenter/scripts/runreflector,
     51     datacenter/scripts/runcamera, datacenter/scripts/setup:
     52     - moved defining of detectorpath from scripts to sourcefile
     53
     54   * datacenter/scripts/runcorsika, datacenter/scripts/setup:
     55     - implemented variable for path of Mmcs
     56
     57   * datacenter/scripts/runcorsika:
     58     - exchanged 'operator' in inputcard by `whoami`
     59     - implemented usage of function for new datastructure
     60
     61   * resources/steps.rc:
     62     - added line for primaries for MCRunProcessingStatus
    3163
    3264

  • trunk/MagicSoft/Mars/datacenter/scripts/jobmanager

    r9122 r9129  
    3737echo "starting jobmanager ("`date`")" >> $jmscriptlog 2>&1
    3838
    39 if [ "$1" == "" ]
    40 then
    41    echo "running jobmanager for data" >> $jmscriptlog 2>&1
    42    scripts=( ${datascripts[@]} )
    43    scriptscolname=( ${datascriptscolname[@]} )
    44 elif [ "$1" = "mc" ]
    45 then
    46    echo "running jobmanager for mc" >> $jmscriptlog 2>&1
    47    scripts=( ${mcscripts[@]} )
    48    scriptscolname=( ${mcscriptscolname[@]} )
    49 else
    50    echo "$1 is awrong commandline option for jobmanager -> exit" >> $jmscriptlog 2>&1
    51    printprocesslog "WARN $1 is wrong commandline option for jobmanager"
    52    finish >> $jmscriptlog 2>&1
    53 fi
     39# decide which jobmanager you want to run
     40# setup of the different jobmanagers (which scripts they start)
     41#  the number of jobs are defined in the file setup
     42case $1 in
     43   data) echo "running jobmanager for data" >> $jmscriptlog 2>&1
     44         scripts=( "runganymed" "runstar" "runcallisto" ) # not used: "dodatacheck" "cutslices"
     45         scriptscolname=( "fGanymed" "fStar" "fCallisto" ) # not used: "fDataCheckDone" "fCompmux"
     46         pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ) # not used: ${pnodatacheck[@]} ${pnocutslices[@]}
     47         pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ) # not used: ${pnodatacheckwe[@]} ${pnocutsliceswe[@]}
     48         break
     49         ;;
     50     mc) echo "running jobmanager for mc" >> $jmscriptlog 2>&1
     51         scripts=( "runcorsika" "runreflector" "runcamera" )
     52         scriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
     53         pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} )
     54         pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )
     55         break
     56         ;;
     57      *) echo "$1 is a wrong commandline option for jobmanager -> exit" >> $jmscriptlog 2>&1
     58         printprocesslog "WARN $1 is wrong commandline option for jobmanager"
     59         finish >> $jmscriptlog 2>&1
     60         break
     61         ;;
     62esac
     63
     64# choose commands according to queueing system (defined in setup)
     65case $queuesys in
     66      sge)  echo "setting commands for sun grid engine" >> $jmscriptlog 2>&1
     67            alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -b y -e `echo $runlogpath`/error-`echo $date`.log -o `echo $runlogpath`/log-`echo $date`.log `echo $scriptspath`/`echo ${scripts[$i]}` '
     68#            alias 'queuesubmit'='/opt/gridengine/bin/lx26-amd64/qsub -sc runlogpath=`echo $runlogpath` -sc date=`echo $date` -sc scriptspath=`echo $scriptspath` -sc script=`echo ${scripts[$i]}` `echo $scriptspath`/job.sge '
     69            alias 'checkqueue'="/opt/gridengine/bin/lx26-amd64/qstat | awk ' { print \"Owner\"\$4\" \" \$3\"Jobstatus\"\$5 } '"
     70            break
     71            ;;
     72   condor)  echo "setting commands for condor" >> $jmscriptlog 2>&1
     73            alias 'queuesubmit'='/usr/local/bin/condor_submit -a path=`echo $scriptspath` -a prog=`echo ${scripts[$i]}` -a date=`echo $date` -a dir=`echo $runlogpath` `echo $scriptspath`/run.condor'
     74            alias 'checkqueue'='/usr/local/bin/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus '
     75            break
     76            ;;
     77        *)  finish >> $jmscriptlog 2>&1
     78            ;;
     79esac
    5480
    5581prev=$max
    … …  
    5985nothingtodocount=0
    6086nothingtodosleeptime=0
     87errorsleeptime=$errorsleeptimedefault
    6188while (( $notcount < 100 ))
    6289do
    … …  
    96123
    97124      # get processes in queue
    98       q=(`/usr/local/bin/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus 2>&1 `)
     125#      q=(`/usr/local/bin/condor_q -global -format "Owner%s " Owner -format "%s" CMD -format "Jobstatus%s\n" Jobstatus 2>&1 `)
     126#      q=(`/opt/gridengine/bin/lx26-amd64/qstat | awk ' { print "Owner"$4" " $3"Jobstatus"$5 } ' 2>&1 `)
     127      q=(`checkqueue 2>&1 `)
    99128      if echo $q | egrep \(Error\|failed\)
    100129      then
    101          echo `date`" WARN condor_q failed" >> $jmscriptlog 2>&1
    102          printprocesslog "WARN condor_q failed"
    103          echo `date`" WARN condor_q failed" >> $jmerrorlog
     130         echo `date`" WARN checking query ($queuesys) failed" >> $jmscriptlog 2>&1
     131         printprocesslog "WARN checking query ($queuesys) failed"
     132         echo `date`" WARN checking query ($queuesys) failed" >> $jmerrorlog
    104133         cont >> $jmscriptlog 2>&1
    105134      fi
    … …  
    111140      queuedscript=${#q2[@]}
    112141      # get running scripts
    113       q3=(`echo ${q[@]} | egrep -o ${scripts[$i]}Jobstatus2`)
     142      q3=( `echo ${q[@]} | egrep -o \(${scripts[$i]}Jobstatus2\|${scripts[$i]}Jobstatusr\)` )
    114143      runningscript=${#q3[@]}
    115144      stillinqueue=`echo $queuedscript - $runningscript | bc `
    … …  
    151180      prev=$max
    152181
    153       # submit 1 script to condor
     182      # submit 1 script to queuing system
    154183      date=`date +%Y-%m-%d`
    155       echo " committing 1 ${scripts[$i]} to condor" >> $jmscriptlog 2>&1
    156       if ! /usr/local/bin/condor_submit -a path=$scriptspath -a prog=${scripts[$i]} -a date=$date -a dir=$runlogpath $scriptspath/run.condor 2>> $jmerrorlog
    157       then
    158          echo `date`" WARN condor_submit failed" >> $jmerrorlog
    159          echo "condor is not working -> sleeping $errorsleeptime" >> $jmscriptlog 2>&1
    160          printprocesslog "WARN submitting ${scripts[$i]} to condor failed"
     184      echo " committing 1 ${scripts[$i]} to $queuesys" >> $jmscriptlog 2>&1
     185#      if ! /usr/local/bin/condor_submit -a path=$scriptspath -a prog=${scripts[$i]} -a date=$date -a dir=$runlogpath $scriptspath/run.condor 2>> $jmerrorlog
     186#      if ! /opt/gridengine/bin/lx26-amd64/qsub -e $runlogpath/error-$date.log -o $runlogpath/log-$(date).log 2>> $jmerrorlog
     187      if ! queuesubmit 2>> $jmerrorlog
     188      then
     189         echo `date`" WARN submitting job ($queuesys) failed" >> $jmerrorlog
     190         echo "$queuesys is not working -> sleeping $errorsleeptime" >> $jmscriptlog 2>&1
     191         printprocesslog "WARN submitting ${scripts[$i]} ($queuesys) failed"
    161192         if [ $errorsleeptime -lt $sleeptimelimit ]
    162193         then

  • trunk/MagicSoft/Mars/datacenter/scripts/runcamera

    r9072 r9129  
    3232program=camera
    3333column=fCameraFileAvail
    34 detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector
    3534
    3635set -C

  • trunk/MagicSoft/Mars/datacenter/scripts/runcorsika

    r9072 r9129  
    3535set -C
    3636
    37 export FLUPRO='/home/operator/Corsika/Mmcs6500/fluka2006_3'
     37export FLUPRO=$corsikapath'/fluka2006_3'
    3838
    3939# printing input card to stdout
    … …  
    7373   echo "CERTEL 1"
    7474   echo " 0. 0. 0. 0. 0. 1800. 1700."
    75    echo "USER operator"
     75   echo "USER "`whoami`
    7676   echo "ATMOSPHERE 11 T"
    7777   echo "EXIT"
    … …  
    8383
    8484# get sequence #
    85 gettodo >> $scriptlog 2>&1
    86 run=$process
     85gettodo "1" >> $scriptlog 2>&1
     86run=${primaries[0]}
    8787# get corsika runno
    8888query="SELECT fCorsikaRunNumber FROM MCRunData where fMCRunNumber="$run
    … …  
    136136makedir $outpath >> $scriptlog 2>&1
    137137
    138 cd /home/operator/Corsika/Mmcs6500/
     138cd $corsikapath
    139139
    140140echo "INPUTCARD:" >> $logfile 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/runreflector

    r9072 r9129  
    3232program=reflector
    3333column=fReflectorFileAvail
    34 detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector
    3534
    3635set -C

  • trunk/MagicSoft/Mars/datacenter/scripts/setup

    r9116 r9129  
    3737fi
    3838
     39# queuing system
     40queusys=condor # wuerzburg data center
     41#queuesys=sge  # isdc
    3942
     43# logging and setup
    4044logpath=/magic/datacenter/autologs
    4145lockpath=/magic/datacenter/locks
    4246setuppath=/magic/datacenter/setup
    4347
     48# paths of data
    4449datapath=/magic/data
    4550subsystempath=/magic/subsystemdata
    … …  
    4752datasetpath=/magic/datasets
    4853
     54# paths and setup for mc
    4955mcpath=/magic/montecarlo
    5056mcrawpath=$mcpath/rawfiles
    5157mcsequpath=$mcpath/sequences
     58corsikapath=/home/operator/Corsika/Mmcs6500/
     59detectordir=/home/msmeyer/MC/MagicSoft/Simulation/Detector
    5260reflectorversion="reflector 0.7"
    53 cameraversion="camera 0.75"
     61cameraversion="camera 0.8"
    5462
    5563webpath=/www/htdocs/datacenter
    … …  
    6674errorsleeptimedefault=60 #60
    6775max=42 #maximum number of processes
    68 
    69 # not done/used
    70 #scripts=( "dodatacheck" "cutslices" )
    71 #scriptscolname=( "fDataCheckDone" "fCompmux" )
    72 # currently used
    73 datascripts=( "runganymed" "runstar" "runcallisto" )
    74 datascriptscolname=( "fGanymed" "fStar" "fCallisto" )
    7576
    7677#
    … …  
    9192#pnocutslices=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    9293#pnodatacheck=(   20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
     94pnocorsika=(     12 12 12 12 12 12 12 10  9  8  7  7  7  7  7  7  7  7  7  8  9 10 12 12 )
     95pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
     96pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
    9397
    9498pnocallistowe=(  32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 )
    … …  
    97101#pnocutsliceswe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    98102#pnodatacheckwe=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
     103pnocorsikawe=(   12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 )
     104pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
     105pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
    99106
    100 # not used/done
    101 #pnosweek=( ${pnodatacheck[@]} ${pnocutslices[@]} )
    102 #pnoswe=( ${pnodatacheckwe[@]} ${pnocutsliceswe[@]} )
    103 # currently used
    104 pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} )
    105 pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} )
    106 
    107 
    108 # not yet working completely
    109 
    110 ## values for mc production
    111 #mcscripts=( "runcorsika" "runreflector" "runcamera" )
    112 #mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
    113 #
    114 ##hour:            0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
    115 #
    116 #pnocorsika=(     12 12 12 12 12 12 12 10  9  8  7  7  7  7  7  7  7  7  7  8  9 10 12 12 )
    117 #pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    118 #pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
    119 #
    120 #pnocorsikawe=(   12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 )
    121 #pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    122 #pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
    123 #
    124 #pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} )
    125 #pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )
    126 

  • trunk/MagicSoft/Mars/datacenter/scripts/sourcefile

    r9122 r9129  
    231231   pw=`grep Password $mars/sql.rc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
    232232   us=`grep User $mars/sql.rc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
     233   ho=`grep URL $mars/sql.rc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
    233234#   echo "setup: "
    234235#   echo " db: "$db
    235236#   echo " pw: "$pw
    236237#   echo " us: "$us
     238#   echo " ho: "$ho
    237239}
    238240
    … …  
    284286   fi
    285287   echo " QUERY: "$query
    286    if ! process=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
     288   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
    287289   then
    288290      echo "ERROR could not query processes from db -> exit"
    … …  
    321323   query=$query" group by $column "
    322324#   echo " QUERY: "$query
    323    if ! numproc=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
     325   if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
    324326   then
    325327      echo "ERROR could not query number of todo proceses from db -> continue"
    … …  
    373375   done
    374376   echo " QUERY: "$query
    375    if ! mysql -s -u $us --password=$pw --host=vela $db -e " $query "
     377   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
    376378   then
    377379      echo "ERROR could not insert status into db -> exit"
    … …  
    386388{
    387389   getdbsetup
    388    if ! val=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
     390   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
    389391   then
    390392      printprocesslog "ERROR could not query db (program: $program, function sendquery)"

  • trunk/MagicSoft/Mars/resources/steps.rc

    r9122 r9129  
    44SequenceProcessStatus.Primary: fSequenceFirst fTelescopeNumber
    55DataSetProcessStatus.Primary: fDataSetNumber
     6MCRunProcessStatus.Primary: fMCRunNumber
    67
    78#--------------------------------------------------------------------
Note: See TracChangeset for help on using the changeset viewer.