source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9221

#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2007
#
#
# ========================================================================
#
# This a resource file for the scripts, in which the standard paths and 
# functions, which are needed more often are stored. 
# Only constant variables are stored here, changing variables are stored 
# in datacenter/scripts/setup
#

# check if script has been started with absolute path
if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
then 
   echo "Please start your script with an absolute path."
   exit
fi

source `dirname $0`/setup 

mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'`
macrospath=$mars/datacenter/macros
scriptspath=$mars/datacenter/scripts

datetime=`date +%F-%H-%M-%S`


# function to make sure that a directory is made
function makedir()
{
   if [ ! -d $@ ]
   then
      mkdir -pv $@ 
      if [ ! -d $@ ]
      then 
         echo "could not make dir "$@
         if ls $lockfile >/dev/null 2>&1
         then 
            rm -v $lockfile
         fi
         date
         exit
      fi
   fi
}

# logging paths for runlogs and processlog
runlogpath=$logpath/run/`date +%Y/%m/%d`
processlogpath=$logpath/processlog
makedir $runlogpath
makedir $processlogpath
processlog=$processlogpath/process`date +%F`.log

makedir $lockpath


# function to provide proper logging in a single logfile ($processlog)
function printprocesslog
{
   makedir $processlogpath
   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
}

# function to exit a script properly
function finish()
{
   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
   then 
      rm -v $lockfile
   fi
   date
   printprocesslog "INFO finished $0"
   exit
}


# set checkvalue to ok at the beginning of the scripts
check="ok"

#failed codes
#sequence build status
Fbuildsequ=1
Fdoexcl=2
#run process status
Ftimecorr=3
Ffillraw=4
Fsinope=5
Ffillsinope=6
Fresetexcl=7
#sequence process status
Fwritesequfile=8
Ffilesavail=9
Fnoccfile=10
Fnocacofile=11
Fmerppcc=12
Fmerppcaco=13
Fcallisto=14
Ffillcalib=15
Ffillsignal=16
Fstar=17
Ffillstar=18
#dataset process status
Fwritedatasetfile=19
Fstardone=20
Fganymed=21
Ffillganymed=22
#again run process status
FCompmux=26
Fdowebplots=27
#again mc run process status
Fmccallisto=28
Ffillmccalib=29
Ffillmcsignal=30
Fmcstar=31
Ffillmcstar=32

# setup for jobmanager:
#   log files (can't be defined in script itself, as script can run longer 
#   than one day
jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
jmscriptlog=$runlogpath/jobmanager`date +%F`.log
steps=$mars/resources/steps.rc
if ! ls $steps >/dev/null
then
   echo "Can't find steps.rc ($steps)"
   finish
fi
sqlrc=$mars/sql.rc
if ! ls $sqlrc >/dev/null
then
   echo "Can't find sql.rc ($sqlrc)"
   finish
fi
# resetting values
pno=0
totalpno=0
running=0
queued=0
runningscript=0
queuedscript=0
stillinqueue=0


# alias (we cannot check the beginning of the line due to
# color codes in filldotraw.C)
alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'

# in the following the functions, which are needed by several scripts, are 
# defined

# function to do continue in a loop and produce according logging
function cont()
{
   date
   echo ""
   continue
}

# function to check if a process is already locked
function checklock()
{
   date > $lockfile 
   checklock0=$?
   case $checklock0 in
      0)   echo " checklock0=$checklock0 -> continue " ;;
      1)   echo " checklock0=$checklock0 -> file $lockfile exists"
           date
           $@
           exit;;
      *)   echo " checklock0=$checklock0 -> something went completely wrong" ;;
   esac
}

function resetstatusvalues()
{
   statustime=NULL
   starttime=NULL
   returncode=NULL
   programid=NULL
   failedtime=NULL
}

function printstatusvalues()
{
   echo "the current values are:"
   echo " statustime=$statustime"
   echo " starttime=$starttime"
   echo " returncode=$returncode"
   echo " programid=$programid"
   echo " failedtime=$failedtime"
   echo "-- check: -$check-"
   echo ""
}

# function evaluating the statusvalues
function evalstatus()
{
   case $@ in
      start)   echo "setstatus start"
               starttime="Now()"
               ;;
       stop)   case $check in
                  ok)  echo "setstatus stop - ok"
                       statustime="Now()"
                       ;;
                  no)  echo "setstatus stop - nothing new"
                       check="ok"
                       ;;
                   *)  echo "setstatus stop - failed"
                       starttime=noreset
                       if [ "$check" == "" ]
                       then
                          returncode=1
                       else
                          returncode=$check
                       fi
                       programid=$com
                       failedtime="Now()"
                       check="ok"
                       ;;
               esac
               ;;
          *)   echo "error -> exit"
               printprocesslog "ERROR function evalstatus got wrong variable"
               finish
               ;;
   esac
}

function getdbsetup()
{
   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
   pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
#   echo "setup: "
#   echo " db: "$db
#   echo " pw: "$pw
#   echo " us: "$us
#   echo " ho: "$ho
}

function getstepinfo()
{
   table=`grep "$column:" $steps | sed -e "s/[.]$column://" -e 's/#//' -e 's/ //g'`
   coltab=`grep "$column:" $steps | sed -e 's/://' -e 's/#//' -e 's/ //g'`
   needs=`grep "$coltab[.]Needs:" $steps | sed -e "s/$coltab[.]Needs://"`
   influences=`grep "$coltab[.]Influences:" $steps | sed -e "s/$coltab[.]Influences://"`
   prims=( `grep "$table[.]Primary:" $steps | sed -e "s/$table[.]Primary://"` )
#   echo " column $column - table $table - coltab $coltab"
#   echo " needs: $needs"
#   echo " influences: $influences"
#   echo " prims: ${prims[@]}"
}

# function to get todolist
function gettodo()
{
   process=
   echo "getting todo..."
   getdbsetup
   getstepinfo
   # get query
   query=" select "${prims[0]}
   for (( i=1 ; i < ${#prims[@]} ; i++ ))
   do
      query=$query", ${prims[$i]}"
   done
   query=$query" from $table where "
   if ! echo $needs | grep '#' > /dev/null
   then
      for need in $needs
      do
         query=$query" not isnull($need) and"
      done
   fi
   query=$query" isnull($column) "
   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fProgramId) and isnull(fReturnCode) "
   query=$query" order by fPriority desc "
   if [ "$@ " != " " ]
   then 
      query=$query" limit 0, $@ "
   fi
   echo " QUERY: "$query
   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      echo "ERROR could not query processes from db -> exit"
      printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
      finish
   fi

   if [ "$process" = "" ]
   then
      echo "  => nothing to do"
      finish
   else
      primaries=( $process )
      num=`expr ${#primaries[@]} / ${#prims[@]} `
   fi
}


# function to get the number of processes which still have to be done
function getstatus()
{
   numproc=
   getdbsetup
   getstepinfo
   # get query
   query=" select count(*) from $table where "
   if ! echo $needs | grep '#' > /dev/null
   then
      for need in $needs
      do
         query=$query" not isnull($need) and"
      done
   fi
   query=$query" isnull($column) "
   query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fProgramId) and isnull(fReturnCode) "
   query=$query" group by $column "
#   echo " QUERY: "$query
   if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      echo "ERROR could not query number of todo proceses from db -> continue"
      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
      cont
   fi
}

# function to set status of a process in the db
function setstatus()
{
   if [ "$column" = "no" ]
   then
      return
   fi
   resetstatusvalues
   evalstatus $@
   getdbsetup
   getstepinfo
   # get query
   reset=`grep "$coltab[.]Reset:" $steps | sed -e "s/$coltab[.]Reset://" -e 's/ //g'`
   if [ "$reset" = "no" ]
   then
      echo "YOU CAN'T RESET $column for ${primaries[$s+$s]}!!!"
      printprocesslog "ERROR you can't reset $column for ${primaries[$s+$s]}"
      finish
   fi
   query=" update $table set $column=$statustime"
   if ! echo $influences | grep '#' > /dev/null
   then
      for influence in $influences
      do
         query=$query", $influence=NULL"
      done
   fi
   if ! [ "$starttime" = "noreset" ]
   then
      query=$query", fStartTime=$starttime"
   fi
   query=$query", fFailedTime=$failedtime, fProgramId=$programid, fReturnCode=$returncode "
   query=$query" where "
   if [ "$s" = "" ]
   then
      s=0
   fi
   query=$query" ${prims[0]}='${primaries[$s*${#prims[@]}]}'"
   for (( i=1 ; i < ${#prims[@]} ; i++ ))
   do
      query=$query" and ${prims[$i]}='${primaries[$s*${#prims[@]}+$i]}' "
   done   
   echo " QUERY: "$query
   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
   then
      echo "ERROR could not insert status into db -> exit"
      printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
      finish
   fi

}

# function to send a mysql query
function sendquery()
{
   getdbsetup
   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      printprocesslog "ERROR could not query db (program: $program, function sendquery)"
      return 1
   fi
   if [ "$val" = "NULL" ]
   then
      val=
   fi
   echo $val
   return 0
}