#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2010
#
#
# ========================================================================
#
# This a resource file for the scripts, in which the standard paths and 
# functions, which are needed more often are stored. 
# Only constant variables are stored here, changing variables are stored 
# in datacenter/scripts/setup
#

# check if script has been started with absolute path
if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
then 
   echo "Please start your script with an absolute path."
   exit
fi

if [ "$AUTOMATIONSETUP" = "" ]
then 
   echo "Please set the environment variable \$AUTOMATIONSETUP."
   exit
fi

# possible solution for $0 problem: 
# ${BASH_SOURCE[0]} 
# but has to be checked and tested more carefully
# should solve at least problem with login-shell and with source
source `dirname $0`/setup.$AUTOMATIONSETUP

datetime=`date +%F-%H-%M-%S`


# function to make sure that a directory is made
function makedir()
{
   if [ ! -d $@ ]
   then
      mkdir -pv $@ 
      if [ ! -d $@ ]
      then 
         if ! [ "$processlog" = "" ]
         then
            echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
         else
            echo "could not make dir "$@
         fi
         if ls $lockfile >/dev/null 2>&1
         then 
            rm -v $lockfile
         fi
         exit
      fi
   fi
}

# logging paths for runlogs and processlog
runlogpath=$logpath/run/`date +%Y/%m/%d`
processlogpath=$logpath/processlog
makedir $runlogpath
makedir $processlogpath
processlog=$processlogpath/process`date +%F`.log

makedir $lockpath


# function to provide proper logging in a single logfile ($processlog)
function printprocesslog
{
   makedir $processlogpath
   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
}

# function to exit a script properly
function finish()
{
   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
   then 
      printprocesslog "INFO " `rm -v $lockfile`
   fi
   printprocesslog "INFO finished $0"
   exit
}


# set checkvalue to ok at the beginning of the scripts
check="ok"

# setup for jobmanager:
#   log files (can't be defined in script itself, as script can run longer 
#   than one day
jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log

# check if rc-files are available
if ! ls $steps >/dev/null
then
   echo "Can't find steps.rc ($steps)"
   finish
fi
if ! ls $sqlrc >/dev/null
then
   echo "Can't find sql.rc ($sqlrc)"
   finish
fi

# resetting values for jobmanager
pno=0
totalpno=0
running=0
queued=0
runningscript=0
queuedscript=0
stillinqueue=0


# alias (we cannot check the beginning of the line due to
# color codes in filldotraw.C)
alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'


# in the following the functions, which are needed by several scripts, are 
# defined

# function to check if a process is already locked
#  command line option can be used to execute something, e.g. 'continue'
function checklock()
{
   if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
   then 
      printprocesslog "WARN lockfile $lockfile exists"
      $@
      exit
   else
      printprocesslog "INFO created lockfile $lockfile"
   fi
}

# print the current status values
function printstatusvalues()
{
   echo "the current values are:"
   echo " starttime=$starttime"
   echo " stoptime=$stoptime"
   echo " returncode=$returncode"
   echo "-- check: -$check-"
   echo ""
}

# get the db-setup from the sql.rc
function getdbsetup()
{
   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
   pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
#   echo "setup: "
#   echo " db: "$db
#   echo " pw: "$pw
#   echo " us: "$us
#   echo " ho: "$ho
}

# function to get the needed information from the dependencies-file steps.rc
function getstepinfo()
{
   getdbsetup
   needs=`grep "$step[.]Needs:" $steps | grep -v '#' | sed -e "s/$step[.]Needs://"`
   noderestricted=`grep "$step[.]NodeRestricted:" $steps | grep -v '#' | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
   prims=( `grep "$step[.]Primaries:" $steps | grep -v '#' | sed -e "s/$step[.]Primaries://"` )
#   echo " needs: $needs"
#   echo " noderestricted: $noderestricted"
#   echo " prims: ${prims[@]}"
}

# function to get the primaries of a step from the dependencies-file steps.rc
function getprimary()
{
   getdbsetup
   grep $@"[.]Primaries:" $steps | grep -v '#' | sed -e "s/$@[.]Primaries://"
}

# function to get the join of a step 
#  from the dependencies-file steps.rc
# $1 gives the step
# $2 gives the type of join
function getjoin()
{
   getdbsetup
   grep $1"[.]"$2"Join:" $steps | grep -v '#' | sed -e "s/$1[.]$2Join://"
}

# function to get the needs of a step from the dependencies-file steps.rc
function getneeds()
{
   getdbsetup
   grep $@"[.]Needs:" $steps | sed -e "s/$@[.]Needs://"
}

# subquery which is needed to check the 
#   status of the steps with different primaries
function subquerypart()
{
   query=$query" SELECT COUNT(*) FROM "$need"Status "`getjoin $need "SpecialInfluence"`
   needinfluences=`grep $need $steps | grep "Needs" | grep -v "$need[.]Needs" | cut -d'.' -f1`
   for needinfl in $needinfluences
   do
      needinfljoin=`getjoin $needinfl "Influence"`
      if ! [ "$needinfljoin" = "" ]
      then
         query=$query" LEFT JOIN "$needinfl"Status USING ("$needinfljoin") "
      fi
      query=$query" "`getjoin $needinfl "SpecialInfluence"`" "
   done
   query=$query" WHERE "`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
   for (( j=1 ; j < ${#prims[@]} ; j++ ))
   do
      query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
   done
}

# function to create the middle part of a query
#  which is identical for the functions getstatus() and gettodo()
function middlepartofquery()
{
   # add from which table the information is queried
   query=$query" FROM "$step"Status "
   # add the joins to the tables in which the status of the preceding steps is stored
   for need in $needs
   do
      needprims=( `getprimary $need` )
      # make sure that the correct joins are used
      specialneedjoin=`getjoin $need "SpecialNeed"`
      if [ "$specialneedjoin" = "" ]
      then
         query=$query" LEFT JOIN "$need"Status USING (${needprims[@]}) "
      else
         query=$query" "$specialneedjoin" "
         query=$query" LEFT JOIN "$need"Status USING (${needprims[@]}) "
         break
      fi
   done
   # add condition
   query=$query" WHERE "
   # add condition for the status of the peceding steps
   counter=0
   specialneedjoin=
   specialinfljoin=
   for need in $needs
   do
      if [ $counter -gt 0 ]
      then
         query=$query" AND "
      fi
      specialneedjoin=$specialneedjoin" "`getjoin $need "SpecialNeed"`
      specialinfljoin=$specialinfljoin" "`getjoin $need "SpecialInfluence"`
      if [ "$specialneedjoin" = " " ]
      then 
         # condition for step with same primaries
         query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
         query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
         query=$query" ISNULL("$need"Status.fReturnCode) "
         specialneedjoin=
         specialinfljoin=
      else 
         # special query to check steps which 
         #   do not have the same primaries
         # comparison of 
         #     number of rows
         #   with
         #     number of rows which are done
         query=$query" ( "
         subquerypart
         query=$query" ) = ( "
         subquerypart
         query=$query" AND NOT ISNULL("$need"Status.fStartTime) "
         query=$query" AND NOT ISNULL("$need"Status.fStopTime) "
         query=$query" AND ISNULL("$need"Status.fReturnCode)) "
      fi
      counter=`echo $counter + 1 | bc -l`
   done
   # add condition for the status of the step itself
   if [ $counter -gt 0 ]
   then
      query=$query" AND "
   fi
   query=$query" ISNULL("$step"Status.fStartTime) "
   query=$query" AND ISNULL("$step"Status.fStopTime) "
   query=$query" AND ISNULL("$step"Status.fReturnCode) "
}

# function to get todolist
#   returns the next or the list of next steps
function gettodo()
{
   # reset the variable for the number of the next step 
   process=
   printprocesslog "INFO getting todo..."
   getstepinfo
   # get query
   query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
   for (( j=1 ; j < ${#prims[@]} ; j++ ))
   do
      query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
   done
   # get middle par of query 
   middlepartofquery
   # add requirement for production host in case it is needed
   if [ "$2 " != " " ]
   then 
      query=$query" AND fProductionHostKEY=$2 "
   fi
   # order by priority to the the number of the next step to be done
   query=$query" ORDER BY "$step"Status.fPriority desc "
   # add limitation in case only one or a limited number of 
   #  processes should be executed
   if [ "$1 " != " " ]
   then 
      query=$query" limit 0, $1 "
   fi
   # print query 
   #echo " gettodo QUERY: "$query
   printprocesslog "INFO gettodo QUERY: "$query
   # execute query 
   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
      finish
   fi
   # get numbers of next step from mysql result
   if [ "$process" = "" ]
   then
      printprocesslog "INFO => nothing to do"
      finish
   else
      primaries=( $process )
      num=`expr ${#primaries[@]} / ${#prims[@]} `
   fi
}

# function to get the number of processes which still have to be done
function getstatus()
{
   # reset the variable for the number of steps to be done
   numproc=
   getstepinfo
   # get query
   query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
   # get middle part of query 
   middlepartofquery
   # add requirement for production host in case it is needed
   if [ "$1 " != " " ]
   then 
      query=$query" AND fProductionHostKEY=$1 "
   fi
   # group by an 'artifical' column to get the number of lines
   query=$query" GROUP BY 2 "
   # printing query
   #echo " getstatus QUERY: "$query
   printprocesslog "INFO getstatus QUERY: "$query
   # execute query
   if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
   then
      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
      echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
      continue
   fi
   # get number of processes from mysql result
   if [ "${numprocs[1]}" = "" ]
   then
      numproc=0
   else
      numproc=${numprocs[0]}
   fi
}

# function to set status of a process in the db
function setstatus()
{
   # remark:
   # this function does not include the 'Default' flag 
   # for resetting steps

   # for dowebplots (there are steps which have no entry in the DB)
   if [ "$step" = "no" ]
   then
      return
   fi
   
   # reset status values
   starttime=NULL
   stoptime=NULL
   returncode=NULL
   # evaluate the status values
   case $@ in
      start)   printprocesslog "INFO setstatus start"
               starttime="Now()"
               ;;
       stop)   case $check in
                  ok)  printprocesslog "INFO setstatus stop - ok"
                       starttime=noreset
                       stoptime="Now()"
                       ;;
                  no)  printprocesslog "INFO setstatus stop - nothing new"
                       check="ok"
                       ;;
                   *)  printprocesslog "INFO setstatus stop - failed"
                       starttime=noreset
                       stoptime="Now()"
                       if [ "$check" == "" ]
                       then
                          returncode=1
                       else
                          returncode=$check
                       fi
                       check="ok"
                       ;;
               esac
               ;;
          *)   printprocesslog "ERROR function setstatus got wrong variable"
               finish
               ;;
   esac
   # get 
   getstepinfo
   
   # get the influences from the steps.rc by evaluating the needs of all steps
   influences=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
   
   # get query
   query=" UPDATE "$step"Status "
   # add joins to the influenced tables
   specialinfljoin=`getjoin $step "SpecialInfluence"`
   if ! [ "$specialinfljoin" = "" ]
   then
      query=$query" "$specialinfljoin
   fi
   for influence in $influences
   do
      infljoin=`getjoin $influence "Influence"`
      if [ "$infljoin" = "" ]
      then
         inflprims=( `getprimary $influence` )
         query=$query" LEFT JOIN "$influence"Status USING (${inflprims[@]}) "
      else 
         query=$query" LEFT JOIN "$influence"Status USING ("$infljoin") "
      fi
      specialinfljoin2=`getjoin $influence "SpecialInfluence"`
      if ! [ "$specialinfljoin2" = "" ]
      then
         query=$query" "$specialinfljoin2
      fi
   done
   # set the status values according to the new status of the step
   query=$query" SET "
   if ! [ "$starttime" = "noreset" ]
   then
      query=$query" "$step"Status.fStartTime=$starttime, "
   fi
   query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
   # set also the status values of the influenced steps
   for influence in $influences
   do
      query=$query", "$influence"Status.fStartTime=NULL "
      query=$query", "$influence"Status.fStopTime=NULL "
      query=$query", "$influence"Status.fReturnCode=NULL "
   done
   # give the condition for which step the status values have to be set
   query=$query" WHERE "
   if [ "$s" = "" ]
   then
      s=0
   fi
   query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
   for (( j=1 ; j < ${#prims[@]} ; j++ ))
   do
      query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
   done   
   # print query
   #echo " setstatus QUERY: "$query
   printprocesslog "INFO setstatus QUERY: "$query
   # execute query
   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
   then
      printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
      finish
   fi
}

# function to send a mysql query
function sendquery()
{
   getdbsetup
   printprocesslog "INFO sendquery QUERY: "$query
   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      printprocesslog "ERROR could not query db (program: $program, function sendquery)"
      return 1
   fi
   if [ "$val" = "NULL" ]
   then
      val=
   fi
   echo $val
   return 0
}