source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile @ 8778

#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2007
#
#
# ========================================================================
#
# This a resource file for the scripts, in which the standard paths and 
# functions, which are needed more often are stored. 
# Only constant variables are stored here, changing variables are stored 
# in datacenter/scripts/setup
#

source `dirname $0`/setup 

mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'`
macrospath=$mars/datacenter/macros
scriptspath=$mars/datacenter/scripts

webpath=/www/htdocs/datacenter

datetime=`date +%F-%H-%M-%S`


# function to make sure that a directory is made
function makedir()
{
   if [ ! -d $@ ]
   then
      mkdir -pv $@ 
      if [ ! -d $@ ]
      then 
         echo "could not make dir "$@
         if ! [ "$todofile" = "" ] && ls $todofile >/dev/null 2>&1
         then 
            rm -v $todofile
         fi
         if ls $lockfile >/dev/null 2>&1
         then 
            rm -v $lockfile
         fi
         date
         exit
      fi
   fi
}

# logging paths for runlogs and processlog
runlogpath=$logpath/run/`date +%Y/%m/%d`
processlogpath=$logpath/processlog
makedir $runlogpath
makedir $processlogpath
processlog=$processlogpath/process`date +%F`.log

makedir $lockpath
makedir $listpath

# set checkvalue to ok at the beginning of the scripts
check="ok"

#failed codes
#sequence build status
Fbuildsequ=1
Fdoexcl=2
#run process status
Ftimecorr=3
Ffillraw=4
Fsinope=5
Ffillsinope=6
Fresetexcl=7
#sequence process status
Fwritesequfile=8
Ffilesavail=9
Fnoccfile=10
Fnocacofile=11
Fmerppcc=12
Fmerppcaco=13
Fcallisto=14
Ffillcalib=15
Ffillsignal=16
Fstar=17
Ffillstar=18
#dataset process status
Fwritedatasetfile=19
Fstardone=20
Fganymed=21
Ffillganymed=22
#mc run process status
Fcorsika=23
Freflector=24
Fcamera=25
#again run process status
FCompmux=26

# setup for jobmanager:
#   log files (can't be defined in script itself, as script can run longer 
#   than one day
jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
jmscriptlog=$runlogpath/jobmanager`date +%F`.log
steps=$mars/steps.rc
# resetting values
pno=0
totalpno=0
running=0
queued=0
runningscript=0
queuedscript=0
stillinqueue=0


# alias (we cannot check the beginning of the line due to
# color codes in filldotraw.C)
alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'

# in the following the functions, which are needed by several scripts, are 
# defined

function printprocesslog
{
   makedir $processlogpath
   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
}

# function to exit a script properly
function finish()
{
   if ! [ "$todofile" = "" ] && ls $todofile >/dev/null 2>&1
   then 
      rm -v $todofile
   fi
   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
   then 
      rm -v $lockfile
   fi
   date
   printprocesslog "INFO finished $0"
   exit
}

# function to do continue in a loop and produce according logging
function cont()
{
   date
   echo ""
   continue
}

# function to check if a process is already locked
function checklock()
{
   date > $lockfile
   checklock0=$?
   case $checklock0 in
      0)   echo " checklock0=$checklock0 -> continue " ;;
      1)   echo " checklock0=$checklock0 -> file $lockfile exists"
           date
           $@
           exit;;
      *)   echo " checklock0=$checklock0 -> something went completely wrong" ;;
   esac
}

function resetstatusvalues()
{
   statustime=NULL
   starttime=NULL
   returncode=NULL
   failedcode=NULL
   failedcodeadd=NULL
   failedtime=NULL
}

function printstatusvalues()
{
   echo "the current values are:"
   echo " statustime=$statustime"
   echo " starttime=$starttime"
   echo " returncode=$returncode"
   echo " failedcode=$failedcode"
   echo " failedcodeadd=$failedcodeadd"
   echo " failedtime=$failedtime"
   echo "-- check: -$check-"
   echo ""
}

# function evaluating the statusvalues
function evalstatus()
{
   case $@ in
      start)   echo "setstatus start"
               starttime="Now()"
               ;;
       stop)   case $check in
                  ok)  echo "setstatus stop - ok"
                       statustime="Now()"
                       ;;
                  no)  echo "setstatus stop - nothing new"
                       check="ok"
                       ;;
                   *)  echo "setstatus stop - failed"
                       starttime=noreset
                       if [ "$check" == "" ]
                       then
                          returncode=1
                       else
                          returncode=$check
                       fi
                       failedcode=$com
                       if ! [ "$comadd" = "" ]
                       then
                          failedcodeadd=$comadd
                       fi
                       failedtime="Now()"
                       check="ok"
                       ;;
               esac
               ;;
          *)   echo "error -> exit"
               printprocesslog "ERROR function evalstatus got wrong variable"
               finish
               ;;
   esac
}

function getdbsetup()
{
   db=`grep Database $mars/sql.rc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
   pw=`grep Password $mars/sql.rc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
   us=`grep User $mars/sql.rc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
#   echo "setup: "
#   echo " db: "$db
#   echo " pw: "$pw
#   echo " us: "$us
}

function getstepinfo()
{
   table=`grep "$column:" $steps | sed -e "s/[.]$column://" -e 's/#//' -e 's/ //g'`
   coltab=`grep "$column:" $steps | sed -e 's/://' -e 's/#//' -e 's/ //g'`
   needs=`grep "$coltab[.]Needs:" $steps | sed -e "s/$coltab[.]Needs://"`
   influences=`grep "$coltab[.]Influences:" $steps | sed -e "s/$coltab[.]Influences://"`
   primary=`grep "$table[.]Primary:" $steps | sed -e "s/$table[.]Primary://"`
#   echo " column $column - table $table - coltab $coltab"
#   echo " needs: $needs"
#   echo " influences: $influences"
#   echo " primary: $primary"
}

# function to get todolist
function getdolist()
{
   echo "getting todolist..."
   getdbsetup
   getstepinfo
   # get query
   query=" select $primary from $table where "
   if ! echo $needs | grep '#' > /dev/null
   then
      for need in $needs
      do
         query=$query" not isnull($need) and"
      done
   fi
   query=$query" isnull($column) "
   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
   query=$query" order by $primary desc "
#   echo " QUERY: "$query
   if ! process=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
   then
      echo "ERROR could not query processes from db -> exit"
      printprocesslog "ERROR could not query processes from db (program: $program, function getdolist)"
      finish
   fi

   if [ "$process" = "" ]
   then
      echo "  => nothing to do"
      finish
   else
      todofile=$listpath/ToDo-$table-$column.txt

      if ls $todofile > /dev/null 2>&1
      then
         echo "$todofile exists already"
         printprocesslog "WARN $todofile exists already (program: $program, function getdolist)"
         finish
      fi
      echo "found processes, writing todofile..."
      echo $process > $todofile
   fi
}

# function to get todo (process)
function gettodo()
{
   process=
   echo "getting todo..."
   getdbsetup
   getstepinfo
   # get query
   query=" select $primary from $table where "
   if ! echo $needs | grep '#' > /dev/null
   then
      for need in $needs
      do
         query=$query" not isnull($need) and"
      done
   fi
   query=$query" isnull($column) "
   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
   query=$query" order by $primary desc "
   query=$query" limit 0, 1 "
#   echo " QUERY: "$query
   if ! process=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
   then
      echo "ERROR could not query process from db -> exit"
      printprocesslog "ERROR could not query process from db (program: $program, function gettodo)"
      finish
   fi

   if [ "$process" = "" ]
   then
      echo "  => nothing to do -> exit"
      finish
   fi
}

# function to get the number of processes which still have to be done
function getstatus()
{
   numproc=
#   echo "getting status..."
   getdbsetup
   column=${scriptscolname[$i]}
   getstepinfo
   # get query
   query=" select count(*) from $table where "
   if ! echo $needs | grep '#' > /dev/null
   then
      for need in $needs
      do
         query=$query" not isnull($need) and"
      done
   fi
   query=$query" isnull($column) "
   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
   query=$query" group by $column "
#   echo " QUERY: "$query
   numproc=0
   if ! numproc=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
   then
      echo "ERROR could not query number of todo proceses from db -> continue"
      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
      cont
   fi
}

# function to set status of a process in the db
function setstatus()
{
   resetstatusvalues
   evalstatus $@
   getdbsetup
   getstepinfo
   # get query
   reset=`grep "$coltab[.]Reset:" $steps | sed -e "s/$coltab[.]Reset://" -e 's/ //g'`
   if [ "$reset" = "no" ]
   then
      echo "YOU CAN'T RESET $column for $primvar!!!"
      printprocesslog "ERROR you can't reset $column for $primvar"
      finish
   fi
   query=" update $table set $column=$statustime"
   if ! echo $influences | grep '#' > /dev/null
   then
      for influence in $influences
      do
         query=$query", $influence=NULL"
      done
   fi
   if ! [ "$starttime" = "noreset" ]
   then
      query=$query", fStartTime=$starttime"
   fi
   query=$query", fFailedTime=$failedtime, fFailedCode=$failedcode, fFailedCodeAdd=$failedcodeadd, fReturnCode=$returncode "
   query=$query" where $primary='$primvar'"
   echo " QUERY: "$query
   if ! mysql -s -u $us --password=$pw --host=vela $db -e " $query "
   then
      echo "ERROR could not insert status into db -> exit"
      printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
      finish
   fi

}

# function to send a mysql query
function sendquery()
{
   getdbsetup
   if ! val=`mysql -s -u $us --password=$pw --host=vela $db -e " $query "`
   then
      printprocesslog "ERROR could not query db (program: $program, function sendquery)"
      return 1
   fi
   if [ "$val" = "NULL" ]
   then
      val=
   fi
   echo $val
   return 0
}