Changeset 7927 for trunk


Ignore:
Timestamp:
08/24/06 12:15:46 (18 years ago)
Author:
Daniela Dorner
Message:
*** empty log message ***
Location:
trunk/MagicSoft/Mars
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/MagicSoft/Mars/Changelog

    r7923 r7927  
    1818
    1919                                                 -*-*- END OF LINE -*-*-
     20 2006/08/24 Daniela Dorner
     21
     22   * datacenter/scripts/sourcefile:
     23     - implemented new functions to get information from the database
     24     - fixed bug in naming of variable
     25
     26   * datacenter/scripts/jobmanager:
     27     - implemented usage of getstatus
     28     - fixed bug in naming of variable
     29
     30   * datacenter/scripts/runcallisto, datacenter/scripts/runstar,
     31     datacenter/scripts/runganymed, datacenter/scripts/dodatacheck:
     32     - implemented usage of gettodo
     33     - removed for-loop to process several primaries
     34
     35   * datacenter/scripts/buildsequenceentries,
     36     datacenter/scripts/checkfilesforsequenceavail,
     37     datacenter/scripts/checkstardone, datacenter/scripts/correcttime,
     38     datacenter/scripts/doexclusions, datacenter/scripts/fillcallisto,
     39     datacenter/scripts/fillganymed, datacenter/scripts/fillstar,
     40     datacenter/scripts/writesequencefiles:
     41     - implemented usage of new getdolist function
     42
     43
     44
    2045 2006/08/23 Thomas Bretz
    2146

  • trunk/MagicSoft/Mars/datacenter/scripts/dodatacheck

    r7920 r7927  
    5151date >> $scriptlog 2>&1
    5252
    53 # get todo file
    54 possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt`  >> $scriptlog 2>&1
    55 if [ "$possibletodofiles" = "" ]
    56 then
    57    echo "ERROR: in $program no todofiles found => something went wrong in jobmanager"
    58    finish >> $scriptlog 2>&1
    59 else
    60    singleprocess="yes"
    61    echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1
    62    for possibletodofile in ${possibletodofiles[@]}
    63    do
    64       if ! ls $possibletodofile >> $scriptlog 2>&1
    65       then
    66          echo "file is not on disk -> continue" >> $scriptlog 2>&1
    67          continue
    68       fi
    69       lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'`
    70       checklock >> $scriptlog 2>&1
    71       todofile=$possibletodofile
    72    done
    73 fi
    74 if [ "$todofile" = "" ]
    75 then
    76    echo "no todofile found -> exit"  >> $scriptlog 2>&1
    77    finish >> $scriptlog 2>&1
    78 fi
     53# get run #
     54gettodo >> $scriptlog 2>&1
     55run=$process
    7956
    80 # get run(s) from todo file
    81 runs=(`cat $todofile`)
    82 if [ "$runs" = "" ]
    83 then
    84    echo "nothing to do -> exit"  >> $scriptlog 2>&1
    85    finish >> $scriptlog 2>&1
    86 fi
    87 echo "runs: "${runs[@]} >> $scriptlog 2>&1
    88 rm -v $todofile >> $scriptlog 2>&1
     57# lock sequ
     58lockfile=$lockpath/lock-$table-$column-$run.txt
     59checklock >> $scriptlog 2>&1
    8960
    9061cd $mars
    9162
    92 # processing run(s)
    93 for run in ${runs[@]}
     63echo "do $program for run "$run >> $scriptlog 2>&1
     64no=`printf %08d $run | cut -c 0-2`
     65no2=`printf %08d $run`
     66var1=$date
     67var2=$no2
     68rawfile=`find /magic/data/rawfiles/ -name *${run}_[D,P,C,S]_*_E.raw -o -name *${run}_[D,P,C,S]_*_E.raw.gz`
     69echo "rawfile: "$rawfile >> $scriptlog 2>&1
     70date=`echo $rawfile | cut -c 22-31`
     71# for sinope the date is needed in the format YYYY-MM-DD
     72date2=`echo $date | sed -e 's/\//-/g'`
     73
     74setstatus "start" >> $scriptlog 2>&1
     75
     76# running filldotraw
     77echo "doing filldotraw..." >> $scriptlog 2>&1
     78filldotrawpath=$logpath/filldotraw/$date
     79makedir $filldotrawpath >> $scriptlog 2>&1
     80filldotrawlogfile=$filldotrawpath/filldotraw-$no2.log
     81
     82check1=`root -q -b $macrospath/filldotraw.C+\("\"$rawfile\""\,kFALSE\) | tee $filldotrawlogfile | grep int | sed -e 's/.*(int)//'`
     83
     84case $check1 in
     85   1)   echo " check1=$check1 -> everything ok -> go on with $program..." >> $scriptlog 2>&1
     86        ;;
     87   0)   echo " check1=$check1 -> no connection to db -> continue..." >> $scriptlog 2>&1
     88        check="no"
     89        setstatus "stop" >> $scriptlog 2>&1
     90        continue ;;
     91   *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     92        com=$Ffillraw
     93        check=$check1
     94        setstatus "stop" >> $scriptlog 2>&1
     95        continue ;;
     96esac
     97
     98# running sinope
     99sinopepath=$datapath/sinope/$date
     100makedir $sinopepath >> $scriptlog 2>&1
     101
     102sins=( "-dat"  "-cal")
     103for sin in ${sins[@]}
    94104do
    95    echo "do $program for run "$run >> $scriptlog 2>&1
    96    no=`printf %08d $run | cut -c 0-2`
    97    no2=`printf %08d $run`
    98    var1=$date
    99    var2=$no2
    100    rawfile=`find /magic/data/rawfiles/ -name *${run}_[D,P,C,S]_*_E.raw -o -name *${run}_[D,P,C,S]_*_E.raw.gz`
    101    echo "rawfile: "$rawfile >> $scriptlog 2>&1
    102    date=`echo $rawfile | cut -c 22-31`
    103    # for sinope the date is needed in the format YYYY-MM-DD
    104    date2=`echo $date | sed -e 's/\//-/g'`
     105   sinopefile=sinope$sin$no2
     106   
     107   echo "running sinope $sin..." >> $scriptlog 2>&1
     108   ./sinope -b -q -v4 -f $sin --outf=$sinopefile --out=$sinopepath --log=$sinopepath/$sinopefile.log --html=$sinopepath/$sinopefile.html --run=$run --date=$date2  2>> $scriptlog > /dev/null
     109   check2=$?
     110   
     111   case $check2 in
     112       0)   echo " check2=$check2 -> everything ok -> go on with $program..." >> $scriptlog 2>&1
     113            ;;
     114       *)   echo " check2=$check2 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     115            com=$Fsinope
     116#            comadd=
     117            check=$check2
     118            setstatus "stop" >> $scriptlog 2>&1
     119            continue 2;;
     120   esac
     121   check2=1
     122done
    105123
    106    setstatus "start" >> $scriptlog 2>&1
     124# running fillsinope
     125fillsinopepath=$logpath/fillsinope/$date
     126makedir $fillsinopepath >> $scriptlog 2>&1
     127fillsinopelogfile=$fillsinopepath/fillsinope-$no2.log
    107128
    108    # running filldotraw
    109    echo "doing filldotraw..." >> $scriptlog 2>&1
    110    filldotrawpath=$logpath/filldotraw/$date
    111    makedir $filldotrawpath >> $scriptlog 2>&1
    112    filldotrawlogfile=$filldotrawpath/filldotraw-$no2.log
     129echo "doing fillsinope..." >> $scriptlog 2>&1
     130check3=`root -q -b $macrospath/fillsinope.C+\($run\,"\"$datapath\""\,kFALSE\) | tee $fillsinopelogfile | grep int | sed -e 's/.*(int)//'`
    113131
    114    check1=`root -q -b $macrospath/filldotraw.C+\("\"$rawfile\""\,kFALSE\) | tee $filldotrawlogfile | grep int | sed -e 's/.*(int)//'`
     132case $check3 in
     133   1)   echo " check3=$check3 -> everything ok -> set status..." >> $scriptlog 2>&1
     134        ;;
     135   0)   echo " check3=$check3 -> no connection to db -> continue..." >> $scriptlog 2>&1
     136        check="no"
     137        setstatus "stop" >> $scriptlog 2>&1
     138        continue ;;
     139   *)   echo " check3=$check3 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     140        com=$Ffillsinope
     141        check=$check3
     142        setstatus "stop" >> $scriptlog 2>&1
     143        continue ;;
     144esac
    115145
    116    case $check1 in
    117       1)   echo " check1=$check1 -> everything ok -> go on with $program..." >> $scriptlog 2>&1
    118            ;;
    119       0)   echo " check1=$check1 -> no connection to db -> continue..." >> $scriptlog 2>&1
    120            check="no"
    121            setstatus "stop" >> $scriptlog 2>&1
    122            continue ;;
    123       *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    124            com=$Ffillraw
    125            check=$check1
    126            setstatus "stop" >> $scriptlog 2>&1
    127            continue ;;
    128    esac
    129    
    130    # running sinope
    131    sinopepath=$datapath/sinope/$date
    132    makedir $sinopepath >> $scriptlog 2>&1
    133    
    134    sins=( "-dat"  "-cal")
    135    for sin in ${sins[@]}
    136    do
    137       sinopefile=sinope$sin$no2
    138      
    139       echo "running sinope $sin..." >> $scriptlog 2>&1
    140       ./sinope -b -q -v4 -f $sin --outf=$sinopefile --out=$sinopepath --log=$sinopepath/$sinopefile.log --html=$sinopepath/$sinopefile.html --run=$run --date=$date2  2>> $scriptlog > /dev/null
    141       check2=$?
    142      
    143       case $check2 in
    144           0)   echo " check2=$check2 -> everything ok -> go on with $program..." >> $scriptlog 2>&1
    145                ;;
    146           *)   echo " check2=$check2 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    147                com=$Fsinope
    148 #               comadd=
    149                check=$check2
    150                setstatus "stop" >> $scriptlog 2>&1
    151                continue 2;;
    152       esac
    153       check2=1
    154    done
    155    
    156    # running fillsinope
    157    fillsinopepath=$logpath/fillsinope/$date
    158    makedir $fillsinopepath >> $scriptlog 2>&1
    159    fillsinopelogfile=$fillsinopepath/fillsinope-$no2.log
     146# resetting the status for exclusions
     147echo "resetting the status for fExclusionsDone for date $date2" >> $scriptlog 2>&1
     148resetlogpath=$logpath/resetexclusions/$date
     149makedir $resetlogpath >> $scriptlog 2>&1
     150resetlog=$resetlogpath/reset-exclusions-$no2-$date2.log
    160151
    161    echo "doing fillsinope..." >> $scriptlog 2>&1
    162    check3=`root -q -b $macrospath/fillsinope.C+\($run\,"\"$datapath\""\,kFALSE\) | tee $fillsinopelogfile | grep int | sed -e 's/.*(int)//'`
     152check5=`root -q -b $macrospath/resetcolumn.C+\("\"fExclusionsDone\""\,"\"SequenceBuildStatus\""\,"\"$date2\""\,"\"$date2\""\,kFALSE\) | tee $resetlog | grep int | sed -e 's/(int)//'`
     153case $check5 in
     154   1)   echo " check5=$check5 -> everything ok, fExclusions have been reset " >> $scriptlog 2>&1
     155        ;;
     156   0)   echo " check5=$check5 -> no connection to db -> continue..." >> $scriptlog 2>&1
     157        check="no"
     158        setstatus "stop" >> $scriptlog 2>&1
     159        continue ;;
     160   *)   echo " check5=$check5 -> ERROR -> step could not be resetted -> repeat step " >> $scriptlog 2>&1
     161        com=$Fresetexcl
     162        comadd=`echo $date2 | sed -e 's/-//g'`
     163        check=$check5
     164        setstatus "stop" >> $scriptlog 2>&1
     165        continue ;;
     166esac
    163167
    164    case $check3 in
    165       1)   echo " check3=$check3 -> everything ok -> set status..." >> $scriptlog 2>&1
    166            ;;
    167       0)   echo " check3=$check3 -> no connection to db -> continue..." >> $scriptlog 2>&1
    168            check="no"
    169            setstatus "stop" >> $scriptlog 2>&1
    170            continue ;;
    171       *)   echo " check3=$check3 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    172            com=$Ffillsinope
    173            check=$check3
    174            setstatus "stop" >> $scriptlog 2>&1
    175            continue ;;
    176    esac
    177 
    178    # resetting the status for exclusions
    179    echo "resetting the status for fExclusionsDone for date $date2" >> $scriptlog 2>&1
    180    resetlogpath=$logpath/resetexclusions/$date
    181    makedir $resetlogpath >> $scriptlog 2>&1
    182    resetlog=$resetlogpath/reset-exclusions-$no2-$date2.log
    183 
    184    check5=`root -q -b $macrospath/resetcolumn.C+\("\"fExclusionsDone\""\,"\"SequenceBuildStatus\""\,"\"$date2\""\,"\"$date2\""\,kFALSE\) | tee $resetlog | grep int | sed -e 's/(int)//'`
    185    case $check5 in
    186       1)   echo " check5=$check5 -> everything ok, fExclusions have been reset " >> $scriptlog 2>&1
    187            ;;
    188       0)   echo " check5=$check5 -> no connection to db -> continue..." >> $scriptlog 2>&1
    189            check="no"
    190            setstatus "stop" >> $scriptlog 2>&1
    191            continue ;;
    192       *)   echo " check5=$check5 -> ERROR -> step could not be resetted -> repeat step " >> $scriptlog 2>&1
    193            com=$Fresetexcl
    194            comadd=`echo $date2 | sed -e 's/-//g'`
    195            check=$check5
    196            setstatus "stop" >> $scriptlog 2>&1
    197            continue ;;
    198    esac
    199 
    200    # set status
    201    statustime="Now()"
    202    failed="NULL"
    203    starttime="NULL"
    204    failedtime="NULL"
    205    var1=$date
    206    var2=$no2
    207    setstatus "stop" >> $scriptlog 2>&1
     168# set status
     169statustime="Now()"
     170failed="NULL"
     171starttime="NULL"
     172failedtime="NULL"
     173var1=$date
     174var2=$no2
     175setstatus "stop" >> $scriptlog 2>&1
    208176done
    209177

  • trunk/MagicSoft/Mars/datacenter/scripts/jobmanager

    r7924 r7927  
    1818#
    1919#
    20 #   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
     20#   Author(s): Daniela Dorner  05/2006 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    2222#   Copyright: MAGIC Software Development, 2000-2006
    … …  
    5656      queuedscript=`/usr/local/bin/condor_q -global | grep $user | grep -c ${scripts[$i]}`
    5757      queuedscript=`setzero $queuedscript`
     58      runningscript=`/usr/local/bin/condor_q -global | grep $user | grep ' R ' | grep -c ${scripts[$i]}`
     59      runningscript=`setzero $runningscript`
     60      stillinqueue=`echo $queuedscript - $runningscript | bc `
    5861
    59       echo "date: "`date`" --- date +%k"`date +%k` >> $jmscriptlog 2>&1
    60       echo "--- hour: $hour --- pnototal(hour): ${pnototal[$hour]} --- totalpno: $totalpno ---" >> $jmscriptlog 2>&1
    6162      totalpno=${pnototal[$hour]}
    62       echo "--- hour: $hour --- pnototal(hour): ${pnototal[$hour]} --- totalpno: $totalpno ---" >> $jmscriptlog 2>&1
    63       echo "date: "`date`" --- date +%k"`date +%k` >> $jmscriptlog 2>&1
    6463     
    6564      #choose array according to the day of the week
    … …  
    7675      if [ "$queued" -gt "$totalpno" ]
    7776      then
    78          contex >> $jmscriptlog 2>&1
     77         cont >> $jmscriptlog 2>&1
    7978      else
    8079         if [ "$queuedscript" -gt "$pnoscript" ]
    8180         then
    82             contex >> $jmscriptlog 2>&1
     81            cont >> $jmscriptlog 2>&1
    8382         fi
    8483      fi
    8584
    86       # get todofile
    87       lockfile=$lockpath/lock-getting-${scripts[$i]}-list.txt
    88       checklock  >> $jmscriptlog 2>&1
    89       singleprocess="yes"
    90       gettodo  >> $jmscriptlog 2>&1
    91       rm -v $lockfile >> $jmscriptlog 2>&1
     85      # check if there's something to do
     86      getstatus
     87      if [ "$numproc" = "" ]
     88      then
     89         cont >> $jmscriptlog 2>&1
     90      fi
     91      if ! [ $numproc -gt 0 ]
     92      then
     93         cont >> $jmscriptlog 2>&1
     94      fi
     95      if [ $numproc -lt $stillinqueue ]
     96      then
     97         echo "numproc($numproc) -lt stillinqueue($stillinqueue)" >> $jmscriptlog 2>&1
     98         cont >> $jmscriptlog 2>&1
     99      fi
    92100
    93101      date=`date +%Y/%m/%d`

  • trunk/MagicSoft/Mars/datacenter/scripts/runcallisto

    r7920 r7927  
    4949#pno=500 # number of processes, i.e. number of todo-files
    5050
    51 lockfile=$lockpath/lock-getting-$program-list.txt
    52 
    5351scriptlogpath=$logpath/run/$program/`date +%Y/%m/%d`
    5452makedir $scriptlogpath
    … …  
    5755date >> $scriptlog 2>&1
    5856
    59 # get todo file
    60 possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt`  >> $scriptlog 2>&1
    61 if [ "$possibletodofiles" = "" ]
    62 then
    63    echo "ERROR: in $program no todofiles found => something went wrong in jobmanager"
    64    finish >> $scriptlog 2>&1
    65 else
    66    singleprocess="yes"
    67    echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1
    68    for possibletodofile in ${possibletodofiles[@]}
    69    do
    70       if ! ls $possibletodofile >> $scriptlog 2>&1
    71       then
    72          echo "file is not on disk -> continue" >> $scriptlog 2>&1
    73          continue
    74       fi
    75       lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'`
    76       checklock >> $scriptlog 2>&1
    77       todofile=$possibletodofile
    78    done
    79 fi
    80 if [ "$todofile" = "" ]
    81 then
    82    echo "no todofile found -> exit"  >> $scriptlog 2>&1
    83    finish >> $scriptlog 2>&1
    84 fi
     57# get sequence #
     58gettodo >> $scriptlog 2>&1
     59sequence=$process
    8560
    86 # get sequence(s) from todo file
    87 sequences=(`cat $todofile`)
    88 if [ "$sequences" = "" ]
    89 then
    90    echo "nothing to do -> exit"  >> $scriptlog 2>&1
    91    finish >> $scriptlog 2>&1
    92 fi
    93 echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    94 rm -v $todofile >> $scriptlog 2>&1
     61# lock sequ
     62lockfile=$lockpath/lock-$table-$column-$sequence.txt
     63checklock >> $scriptlog 2>&1
    9564
    9665cd $mars
    9766
    98 # run calibration for sequence(s)
    99 for sequence in ${sequences[@]}
    100 do
    101   echo "run $program for sequence $sequence..." >> $scriptlog 2>&1
    102   no=`printf %08d $sequence | cut -c 0-4`
    103   no2=`printf %08d $sequence`
    104   var1=$no
    105   var2=$no2
    106   outpath="$datapath/$program/$no/$no2"
    107   makedir $outpath >> $scriptlog 2>&1
     67# run calibration for sequence
     68echo "run $program for sequence $sequence..." >> $scriptlog 2>&1
     69no=`printf %08d $sequence | cut -c 0-4`
     70no2=`printf %08d $sequence`
     71var1=$no
     72var2=$no2
     73outpath="$datapath/$program/$no/$no2"
     74makedir $outpath >> $scriptlog 2>&1
    10875
    109   sequfile="$sequpath/$no/sequence$no2.txt"
    110  
    111   # find callisto.rc file
    112   if [ -e $outpath/callisto.rc ]
    113   then
    114      echo "found $program.rc in $outpath -> using this " >> $scriptlog 2>&1
    115      callistorcseq=$outpath/callisto.rc
    116   else
    117      echo "no $program.rc found in $outpath -> making link " >> $scriptlog 2>&1
    118      if [ 49735 -lt $sequence ] && [ $sequence -lt 53516 ]
    119      then
    120         ln -vs $callistorcmarapr05 $outpath/callisto.rc >> $scriptlog 2>&1
    121      else
    122         ln -vs $callistorcnew $outpath/callisto.rc >> $scriptlog 2>&1
    123      fi
    124      callistorcseq=$outpath/callisto.rc
    125   fi
     76sequfile="$sequpath/$no/sequence$no2.txt"
    12677
    127   setstatus "start" >> $scriptlog 2>&1
     78# find callisto.rc file
     79if [ -e $outpath/callisto.rc ]
     80then
     81   echo "found $program.rc in $outpath -> using this " >> $scriptlog 2>&1
     82   callistorcseq=$outpath/callisto.rc
     83else
     84   echo "no $program.rc found in $outpath -> making link " >> $scriptlog 2>&1
     85   if [ 49735 -lt $sequence ] && [ $sequence -lt 53516 ]
     86   then
     87      ln -vs $callistorcmarapr05 $outpath/callisto.rc >> $scriptlog 2>&1
     88   else
     89      ln -vs $callistorcnew $outpath/callisto.rc >> $scriptlog 2>&1
     90   fi
     91   callistorcseq=$outpath/callisto.rc
     92fi
    12893
    129   ./callisto -b -q -v4 -f -raw --log=$outpath/$program$no2.log --html=$outpath/$program$no2.html --out=$outpath --config=$callistorcseq $sequfile  2>> $scriptlog > /dev/null
    130   check1=$?
     94setstatus "start" >> $scriptlog 2>&1
    13195
    132   case $check1 in
    133      0)   echo " check1=$check1 -> everything ok -> doing update..." >> $scriptlog 2>&1
    134           # running merpp update if calibration worked
    135           # finding files, which have to be updated
    136           echo "finding files to be updated..." >> $scriptlog 2>&1
    137           calfiles=`find $outpath -name *_Y_* `
    138           echo " files to be updated: "$calfiles >> $scriptlog 2>&1
    139           if [ "$calfiles" = "" ]
    140           then
    141              echo " no files found -> continue with next sequence" >> $scriptlog 2>&1
    142              continue
    143           fi
    144          
    145           merpplogpath=$outpath"/merpplogs"
    146           makedir $merpplogpath >> $scriptlog 2>&1
    147          
    148           # updated calibrated data files with the information from the cc and caco files
    149           for calfile in ${calfiles[@]}
    150           do
    151              echo "calfile: "$calfile >> $scriptlog 2>&1
    152              # find cc and caco file
    153              # if file is missing continue with next sequence
    154              runno=`echo $calfile | cut -d_ -f2 | sed -e 's/^0//' | sed -e 's/^0//' | sed -e 's/^0//' `
    155              ccfile=`find /magic/subsystemdata/cc/ -name [2][0][0-2][0-9][0,1][0-9][0-3][0-9]_*${runno}_[P,D,C,S]_*_S.rep`
    156              source=`echo $ccfile | cut -d_ -f4`
    157              cacofile=`find /magic/subsystemdata/caco/ -name dc_[2][0][0-2][0-9]_[0,1][0-9]_[0-3][0-9]_*${runno}_${source}.txt`
    158 #             cacofile=`find /magic/subsystemdata/caco/ -name dc_[2][0][0-2][0-9]_[0,1][0-9]_[0-3][0-9]_*${runno}_*.txt`
    159              if [ "$ccfile" = "" ]
    160              then
    161                 echo "no ccfile found for run "$runno >> $scriptlog 2>&1
    162                 com=$Fnoccfile
    163                 comadd=$runno
    164                 check=0
    165                 break
    166              fi
    167              if [ "$cacofile" = "" ]
    168              then
    169                 echo "cacofile with no $runno not found" >> $scriptlog 2>&1
    170                 echo "finding cacofile..." >> $scriptlog 2>&1
    171                 for (( i = 0; i <= 10; i++ ))
    172                 do
    173                    newrun=`echo $runno - $i | bc`
    174             #      echo "$missingcacorun + $i = $newrun"
    175                    path=`dirname $ccfile`
    176                    path=`echo $path | sed -e 's/cc/caco/'`
    177                    echo "path: "$path >> $scriptlog 2>&1
    178                    cacofile=`find $path -name *$newrun*`
    179                    if [ "$cacofile" = "" ]
    180                    then
    181                       if [ $i -eq 9 ]
    182                       then
    183                          echo "no cacofile found" >> $scriptlog 2>&1
    184                          com=$Fnocacofile
    185                          comadd=$runno
    186                          check=0
    187                       fi   
    188                       continue
    189                    else
    190                       echo "cacofile: "$cacofile >> $scriptlog 2>&1
    191                       break
    192                    fi
    193                 done
    194              fi
    195              ./merpp -u --log=$merpplogpath/merppccupdate$runno.log --html=$merpplogpath/merppccupdate$runno.html --auto-time-stop --runfile=$runno $ccfile $calfile 2>> $scriptlog> /dev/null
    196              check2=$?
    197              case $check2 in
    198                      0)   echo " check2=$check2 -> everything ok, merppccupdate worked -> continue" >> $scriptlog 2>&1;;
    199                      *)   echo " check2=$check2 -> ERROR -> merppccupdate failed" >> $scriptlog 2>&1
    200                           com=$Fmerppcc
    201                           comadd=$runno
    202                           check=$check2
    203                           break ;;
    204              esac
    205              ./merpp -u --log=$merpplogpath/merppcacoupdate$runno.log --html=$merpplogpath/merppcacoupdate$runno.html --auto-time $cacofile $calfile 2>> $scriptlog> /dev/null
    206              check3=$?
    207              case $check3 in
    208                      0)   echo " check3=$check3 -> everything ok, merppcacoupdate worked -> continue" >> $scriptlog 2>&1;;
    209                      *)   echo " check3=$check3 -> ERROR -> merppcacoupdate failed" >> $scriptlog 2>&1
    210                           com=$Fmerppcaco
    211                           comadd=$runno
    212                           check=$check3
    213                           break ;;
    214              esac
    215           done
    216           ;;
    217      *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    218           com=$Fcallisto
    219           check=$check1
    220           ;;
    221   esac
    222  
    223   setstatus "stop"  >> $scriptlog 2>&1
    224 done
     96./callisto -b -q -v4 -f -raw --log=$outpath/$program$no2.log --html=$outpath/$program$no2.html --out=$outpath --config=$callistorcseq $sequfile  2>> $scriptlog > /dev/null
     97check1=$?
     98
     99case $check1 in
     100   0)   echo " check1=$check1 -> everything ok -> doing update..." >> $scriptlog 2>&1
     101        # running merpp update if calibration worked
     102        # finding files, which have to be updated
     103        echo "finding files to be updated..." >> $scriptlog 2>&1
     104        calfiles=`find $outpath -name *_Y_* `
     105        echo " files to be updated: "$calfiles >> $scriptlog 2>&1
     106        if [ "$calfiles" = "" ]
     107        then
     108           echo " no files found -> continue with next sequence" >> $scriptlog 2>&1
     109           continue
     110        fi
     111       
     112        merpplogpath=$outpath"/merpplogs"
     113        makedir $merpplogpath >> $scriptlog 2>&1
     114       
     115        # updated calibrated data files with the information from the cc and caco files
     116        for calfile in ${calfiles[@]}
     117        do
     118           echo "calfile: "$calfile >> $scriptlog 2>&1
     119           # find cc and caco file
     120           # if file is missing continue with next sequence
     121           runno=`echo $calfile | cut -d_ -f2 | sed -e 's/^0//' | sed -e 's/^0//' | sed -e 's/^0//' `
     122           ccfile=`find /magic/subsystemdata/cc/ -name [2][0][0-2][0-9][0,1][0-9][0-3][0-9]_*${runno}_[P,D,C,S]_*_S.rep`
     123           source=`echo $ccfile | cut -d_ -f4`
     124           cacofile=`find /magic/subsystemdata/caco/ -name dc_[2][0][0-2][0-9]_[0,1][0-9]_[0-3][0-9]_*${runno}_${source}.txt`
     125#            cacofile=`find /magic/subsystemdata/caco/ -name dc_[2][0][0-2][0-9]_[0,1][0-9]_[0-3][0-9]_*${runno}_*.txt`
     126           if [ "$ccfile" = "" ]
     127           then
     128              echo "no ccfile found for run "$runno >> $scriptlog 2>&1
     129              com=$Fnoccfile
     130              comadd=$runno
     131              check=0
     132              break
     133           fi
     134           if [ "$cacofile" = "" ]
     135           then
     136              echo "cacofile with no $runno not found" >> $scriptlog 2>&1
     137              echo "finding cacofile..." >> $scriptlog 2>&1
     138              for (( i = 0; i <= 10; i++ ))
     139              do
     140                 newrun=`echo $runno - $i | bc`
     141          #      echo "$missingcacorun + $i = $newrun"
     142                 path=`dirname $ccfile`
     143                 path=`echo $path | sed -e 's/cc/caco/'`
     144                 echo "path: "$path >> $scriptlog 2>&1
     145                 cacofile=`find $path -name *$newrun*`
     146                 if [ "$cacofile" = "" ]
     147                 then
     148                    if [ $i -eq 9 ]
     149                    then
     150                       echo "no cacofile found" >> $scriptlog 2>&1
     151                       com=$Fnocacofile
     152                       comadd=$runno
     153                       check=0
     154                    fi   
     155                    continue
     156                 else
     157                    echo "cacofile: "$cacofile >> $scriptlog 2>&1
     158                    break
     159                 fi
     160              done
     161           fi
     162           ./merpp -u --log=$merpplogpath/merppccupdate$runno.log --html=$merpplogpath/merppccupdate$runno.html --auto-time-stop --runfile=$runno $ccfile $calfile 2>> $scriptlog> /dev/null
     163           check2=$?
     164           case $check2 in
     165                   0)   echo " check2=$check2 -> everything ok, merppccupdate worked -> continue" >> $scriptlog 2>&1;;
     166             *)   echo " check2=$check2 -> ERROR -> merppccupdate failed" >> $scriptlog 2>&1
     167                        com=$Fmerppcc
     168                        comadd=$runno
     169                        check=$check2
     170                        break ;;
     171           esac
     172           ./merpp -u --log=$merpplogpath/merppcacoupdate$runno.log --html=$merpplogpath/merppcacoupdate$runno.html --auto-time $cacofile $calfile 2>> $scriptlog> /dev/null
     173           check3=$?
     174           case $check3 in
     175                   0)   echo " check3=$check3 -> everything ok, merppcacoupdate worked -> continue" >> $scriptlog 2>&1;;
     176             *)   echo " check3=$check3 -> ERROR -> merppcacoupdate failed" >> $scriptlog 2>&1
     177                        com=$Fmerppcaco
     178                        comadd=$runno
     179                        check=$check3
     180                        break ;;
     181           esac
     182        done
     183          ;;
     184   *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     185        com=$Fcallisto
     186        check=$check1
     187        ;;
     188esac
     189
     190setstatus "stop"  >> $scriptlog 2>&1
    225191
    226192finish >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/runganymed

    r7920 r7927  
    4343
    4444column=fGanymed
    45 #pno=24 # number of processes, i.e. number of todo-files
    46 
    47 lockfile=$lockpath/lock-getting-$program-list.txt
    4845
    4946scriptlogpath=$logpath/run/$program/`date +%Y/%m/%d`
    … …  
    5350date >> $scriptlog 2>&1
    5451
    55 # get todo file
    56 possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt`  >> $scriptlog 2>&1
    57 if [ "$possibletodofiles" = "" ]
     52# get dataset #
     53gettodo >> $scriptlog 2>&1
     54dataset=$process
     55
     56# lock sequ
     57lockfile=$lockpath/lock-$table-$column-$dataset.txt
     58checklock >> $scriptlog 2>&1
     59
     60cd $mars
     61
     62echo "run $program for dataset $dataset..." >> $scriptlog 2>&1
     63no=`printf %08d $dataset | cut -c 0-5`
     64no2=`printf %08d $dataset`
     65var1=$no
     66var2=$no2
     67outpath="$datapath/$program/$no/$no2"
     68makedir $outpath >> $scriptlog 2>&1
     69
     70datasetfile="$datasetpath/$no/dataset$no2.txt"
     71# get observation mode to choose ganymed.rc file
     72wobble=`grep 'WobbleMode:' $datasetfile` >> $scriptlog 2>&1
     73wobble2=`echo $wobble | grep ^\#` >> $scriptlog 2>&1
     74if [ "$wobble2" = "" ]
    5875then
    59    echo "ERROR: in $program no todofiles found => something went wrong in jobmanager"
    60    finish >> $scriptlog 2>&1
     76   mode="wobble" >> $scriptlog 2>&1
    6177else
    62    singleprocess="yes"
    63    echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1
    64    for possibletodofile in ${possibletodofiles[@]}
    65    do
    66       if ! ls $possibletodofile >> $scriptlog 2>&1
    67       then
    68          echo "file is not on disk -> continue" >> $scriptlog 2>&1
    69          continue
    70       fi
    71       lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'`
    72       checklock >> $scriptlog 2>&1
    73       todofile=$possibletodofile
    74    done
     78   mode="onoff" >> $scriptlog 2>&1
    7579fi
    76 if [ "$todofile" = "" ]
    77 then
    78    echo "no todofile found -> exit"  >> $scriptlog 2>&1
    79    finish >> $scriptlog 2>&1
    80 fi
     80ganymedrc=$setuppath/ganymed/ganymed_$mode.rc
    8181
    82 # retrieve dataset from todo file
    83 datasets=(`cat $todofile`)
    84 if [ "$datasets" = "" ]
    85 then
    86    echo "nothing to do -> exit"  >> $scriptlog 2>&1
    87    finish >> $scriptlog 2>&1
    88 fi
    89 echo "datasets: "${datasets[@]} >> $scriptlog 2>&1
    90 rm -v $todofile >> $scriptlog 2>&1
     82setstatus "start" >> $scriptlog 2>&1
    9183
    92 # run ganymed for dataset
    93 for dataset in ${datasets[@]}
    94 do
    95   echo "run $program for dataset $dataset..." >> $scriptlog 2>&1
    96   no=`printf %08d $dataset | cut -c 0-5`
    97   no2=`printf %08d $dataset`
    98   var1=$no
    99   var2=$no2
    100   outpath="$datapath/$program/$no/$no2"
    101   makedir $outpath >> $scriptlog 2>&1
     84./ganymed -b -q -v4 -f --ind=$datapath/star --config=$ganymedrc --log=$outpath/$program$no2.log --html=$outpath/$program$no2.html --out=$outpath $datasetfile  2>> $scriptlog> /dev/null
     85check1=$?
    10286
    103   datasetfile="$datasetpath/$no/dataset$no2.txt"
    104   # get observation mode to choose ganymed.rc file
    105   wobble=`grep 'WobbleMode:' $datasetfile` >> $scriptlog 2>&1
    106   wobble2=`echo $wobble | grep ^\#` >> $scriptlog 2>&1
    107   if [ "$wobble2" = "" ]
    108   then
    109      mode="wobble" >> $scriptlog 2>&1
    110   else
    111      mode="onoff" >> $scriptlog 2>&1
    112   fi
    113   ganymedrc=$setuppath/ganymed/ganymed_$mode.rc
    114  
    115   setstatus "start" >> $scriptlog 2>&1
     87case $check1 in
     88   0)   echo " check1=$check1 -> everything ok " >> $scriptlog 2>&1 ;;
     89   *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     90        com=$Fganymed
     91        check=$check1
     92        ;;
     93esac
    11694
    117   ./ganymed -b -q -v4 -f --ind=$datapath/star --config=$ganymedrc --log=$outpath/$program$no2.log --html=$outpath/$program$no2.html --out=$outpath $datasetfile  2>> $scriptlog> /dev/null
    118   check1=$?
    119 
    120   case $check1 in
    121      0)   echo " check1=$check1 -> everything ok " >> $scriptlog 2>&1 ;;
    122      *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    123           com=$Fganymed
    124           check=$check1
    125           ;;
    126   esac
    127  
    128   setstatus "stop" >> $scriptlog 2>&1
    129 done
     95setstatus "stop" >> $scriptlog 2>&1
    13096
    13197finish >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/runstar

    r7920 r7927  
    4343
    4444column=fStar
    45 #pno=500 # number of processes, i.e. number of todo-files
    46 
    47 lockfile=$lockpath/lock-getting-$program-list.txt
    4845
    4946scriptlogpath=$logpath/run/$program/`date +%Y/%m/%d`
    … …  
    5350date >> $scriptlog 2>&1
    5451
    55 # get todo file
    56 possibletodofiles=`ls -r $listpath/ToDo-*-$column-*.txt`  >> $scriptlog 2>&1
    57 if [ "$possibletodofiles" = "" ]
    58 then
    59    echo "ERROR: in $program no todofiles found => something went wrong in jobmanager"
    60    finish >> $scriptlog 2>&1
    61 else
    62    singleprocess="yes"
    63    echo "todofiles: "${possibletodofiles[@]} >> $scriptlog 2>&1
    64    for possibletodofile in ${possibletodofiles[@]}
    65    do
    66       if ! ls $possibletodofile >> $scriptlog 2>&1
    67       then
    68          echo "file is not on disk -> continue" >> $scriptlog 2>&1
    69          continue
    70       fi
    71       lockfile=`echo $possibletodofile | sed -e 's/lists/locks/' -e 's/ToDo/lock/'`
    72       checklock >> $scriptlog 2>&1
    73       todofile=$possibletodofile
    74    done
    75 fi
    76 if [ "$todofile" = "" ]
    77 then
    78    echo "no todofile found -> exit"  >> $scriptlog 2>&1
    79    finish >> $scriptlog 2>&1
    80 fi
     52# get sequence #
     53gettodo >> $scriptlog 2>&1
     54sequence=$process
    8155
    82 # retrieve sequence from todo file
    83 sequences=(`cat $todofile`)
    84 if [ "$sequences" = "" ]
    85 then
    86    echo "nothing to do -> exit"  >> $scriptlog 2>&1
    87    finish >> $scriptlog 2>&1
    88 fi
    89 echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    90 rm -v $todofile >> $scriptlog 2>&1
     56# lock sequ
     57lockfile=$lockpath/lock-$table-$column-$sequence.txt
     58checklock >> $scriptlog 2>&1
    9159
    9260cd $mars
    9361
    94 # run star for sequence
    95 for sequence in ${sequences[@]}
    96 do
    97   echo "run $program for sequence $sequence..." >> $scriptlog 2>&1
    98   no=`printf %08d $sequence | cut -c 0-4`
    99   no2=`printf %08d $sequence`
    100   var1=$no
    101   var2=$no2
    102   outpath="$datapath/$program/$no/$no2"
    103   inpath=`echo $outpath | sed -e 's/star/callisto/'`
    104   makedir $outpath >> $scriptlog 2>&1
     62echo "run $program for sequence $sequence..." >> $scriptlog 2>&1
     63no=`printf %08d $sequence | cut -c 0-4`
     64no2=`printf %08d $sequence`
     65var1=$no
     66var2=$no2
     67outpath="$datapath/$program/$no/$no2"
     68inpath=`echo $outpath | sed -e 's/star/callisto/'`
     69makedir $outpath >> $scriptlog 2>&1
    10570
    106   sequfile="$sequpath/$no/sequence$no2.txt"
    107  
    108   setstatus "start" >> $scriptlog 2>&1
     71sequfile="$sequpath/$no/sequence$no2.txt"
    10972
    110   ./star -b -q -v4 -f --log=$outpath/$program$no2.log --html=$outpath/$program$no2.html --ind=$inpath --out=$outpath $sequfile  2>> $scriptlog> /dev/null
    111   check1=$?
     73setstatus "start" >> $scriptlog 2>&1
    11274
    113   case $check1 in
    114      0)   echo " check1=$check1 -> everything ok " >> $scriptlog 2>&1;;
    115      *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    116           com=$Fstar
    117           check=$check1
    118           ;;
    119   esac
    120  
    121   setstatus "stop" >> $scriptlog 2>&1
    122 done
     75./star -b -q -v4 -f --log=$outpath/$program$no2.log --html=$outpath/$program$no2.html --ind=$inpath --out=$outpath $sequfile  2>> $scriptlog> /dev/null
     76check1=$?
     77
     78case $check1 in
     79   0)   echo " check1=$check1 -> everything ok " >> $scriptlog 2>&1;;
     80   *)   echo " check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     81        com=$Fstar
     82        check=$check1
     83        ;;
     84esac
     85
     86setstatus "stop" >> $scriptlog 2>&1
    12387
    12488finish >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/sourcefile

    r7925 r7927  
    7070Ffillganymed=21
    7171
     72#
    7273# setup for jobmanager
     74#
    7375steps=$mars/steps.rc
    74 
     76# resetting values
    7577pno=0
    7678totalpno=0
    … …  
    7981runningscript=0
    8082queuedscript=0
    81 
     83stillinqueue=0
    8284
    8385function setzero()
    … …  
    100102
    101103
    102 
    103104# in the following the function, which are needed by several scripts, are
    104105# defined
    … …  
    113114}
    114115
    115 # function to do continue in a loop or exit the program depending on what is needed
    116 function contex()
    117 {
     116# function to do continue in a loop and produce according logging
     117function cont()
     118{
     119   date
    118120   echo ""
    119    if [ "$singleprocess" = "yes" ]
    120    then
    121       continue
    122    else
    123       finish
    124    fi
     121   continue
    125122}
    126123
    … …  
    148145       1)   echo " checklock0=$checklock0 -> file $lockfile exists -> exit"
    149146            date
    150             contex;;
     147            exit;;
    151148       *)   echo " checklock0=$checklock0 -> something went completely wrong" ;;
    152149   esac
    … …  
    154151
    155152# function calling the macro, which is producing the todo-list
    156 function getdolist()
     153function getdolistroot()
    157154{
    158155   datetime=`date +%F-%H-%M-%S`
    … …  
    278275
    279276# function to get todolist
    280 function gettodo()
    281 {
    282    echo "getting todo..."
     277function getdolist()
     278{
     279   echo "getting todolist..."
    283280   getdbsetup
    284    if [ "$singleprocess" = "yes" ]
    285    then
    286       column=${scriptscolname[$i]}
    287    fi
    288281   getstepinfo
    289282   # get query
    … …  
    296289      done
    297290   fi
    298    if ! echo $influences | grep '#' > /dev/null
    299    then
    300       for influence in $influences
    301       do
    302          query=$query" isnull($influence) and"
    303       done
    304    fi
    305291   query=$query" isnull($column) "
    306292   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
    307293   query=$query" order by $primary desc "
    308    if [ "$singleprocess" = "yes" ]
    309    then
    310       query=$query" limit 0, 1 "
    311    fi
    312294#   echo " QUERY: "$query
    313295   if ! process=`mysql -s -u $us --password=$pw --host=hercules $db -e " $query "`
    314296   then
    315       echo "ERROR could not query process from db -> contex"
    316       rm -v $lockfile
    317       contex
     297      echo "ERROR could not query process from db -> exit"
     298      finish
    318299   fi
    319300
    … …  
    321302   then
    322303      echo "  => nothing to do"
    323       rm -v $lockfile
    324       contex
     304      finish
    325305   else
    326       if [ "$singleprocess" = "yes" ]
    327       then
    328          todofile=$listpath/ToDo-$table-$column-$process.txt
    329          echo "  => found $process"
    330       else
    331          todofile=$listpath/ToDo-$table-$column.txt
    332          echo "found processes"
    333       fi
    334 #      echo " list: "$todofile
     306      todofile=$listpath/ToDo-$table-$column.txt
    335307     
    336308      if ls $todofile > /dev/null 2>&1
    337309      then
    338310         echo "$todofile exists already"
    339          rm -v $lockfile
    340          contex
     311         finish
    341312      fi
     313      echo "found processes, writing todofile..."
    342314      echo $process > $todofile
     315   fi
     316}
     317
     318# function to get todo (process)
     319function gettodo()
     320{
     321   process=
     322   echo "getting todo..."
     323   getdbsetup
     324   getstepinfo
     325   # get query
     326   query=" select $primary from $table where "
     327   if ! echo $needs | grep '#' > /dev/null
     328   then
     329      for need in $needs
     330      do
     331         query=$query" not isnull($need) and"
     332      done
     333   fi
     334   query=$query" isnull($column) "
     335   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
     336   query=$query" order by $primary desc "
     337   query=$query" limit 0, 1 "
     338#   echo " QUERY: "$query
     339   if ! process=`mysql -s -u $us --password=$pw --host=hercules $db -e " $query "`
     340   then
     341      echo "ERROR could not query process from db -> exit"
     342      finish
     343   fi
     344
     345   if [ "$process" = "" ]
     346   then
     347      echo "  => nothing to do -> exit"
     348      finish
     349   fi
     350}
     351
     352# function to get the number of processes which still have to be done
     353function getstatus()
     354{
     355   numproc=
     356   echo "getting status..."
     357   getdbsetup
     358   column=${scriptscolname[$i]}
     359   getstepinfo
     360   # get query
     361   query=" select count(*) from $table where "
     362   if ! echo $needs | grep '#' > /dev/null
     363   then
     364      for need in $needs
     365      do
     366         query=$query" not isnull($need) and"
     367      done
     368   fi
     369   query=$query" isnull($column) "
     370   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
     371   query=$query" group by $column "
     372#   echo " QUERY: "$query
     373   if ! numproc=`mysql -s -u $us --password=$pw --host=hercules $db -e " $query "`
     374   then
     375      echo "ERROR could not query number of todo proceses from db -> continue"
     376      cont
    343377   fi
    344378}
    … …  
    349383   resetstatusvalues
    350384   evalstatus $@
    351 
    352    echo "setting todo..."
    353385   getdbsetup
    354386   getstepinfo
    355387   # get query
    356388   reset=`grep "$coltab[.]Reset:" $steps | sed -e "s/$coltab[.]Reset://" -e 's/ //g'`
    357    echo -$reset-
    358389   if [ "$reset" = "no" ]
    359390   then
Note: See TracChangeset for help on using the changeset viewer.