#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2010
#
#
# ========================================================================
#
# This a resource file for the scripts, in which the standard paths and 
# functions, which are needed more often are stored. 
# Only constant variables are stored here, changing variables are stored 
# in datacenter/scripts/setup
#

# check if script has been started with absolute path
if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
then 
   echo "Please start your script with an absolute path."
   exit
fi

if [ "$AUTOMATIONSETUP" = "" ]
then 
   echo "Please set the environment variable \$AUTOMATIONSETUP."
   exit
fi

# possible solution for $0 problem: 
# ${BASH_SOURCE[0]} 
# but has to be checked and tested more carefully
# should solve at least problem with login-shell and with source
source `dirname $0`/setup.$AUTOMATIONSETUP

datetime=`date +%F-%H-%M-%S`


# function to make sure that a directory is made
function makedir()
{
   if [ ! -d $@ ]
   then
      mkdir -pv $@ 
      if [ ! -d $@ ]
      then 
         if ! [ "$processlog" = "" ]
         then
            echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
         else
            echo "could not make dir "$@
         fi
         if ls $lockfile >/dev/null 2>&1
         then 
            rm -v $lockfile
         fi
         exit
      fi
   fi
}

# logging paths for runlogs and processlog
runlogpath=$logpath/run/`date +%Y/%m/%d`
processlogpath=$logpath/processlog
makedir $runlogpath
makedir $processlogpath
processlog=$processlogpath/process`date +%F`.log

makedir $lockpath


# function to provide proper logging in a single logfile ($processlog)
function printprocesslog
{
   makedir $processlogpath
   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
}

# function to exit a script properly
function finish()
{
   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
   then 
      printprocesslog "INFO " `rm -v $lockfile`
   fi
   printprocesslog "INFO finished $0"
   exit
}


# set checkvalue to ok at the beginning of the scripts
check="ok"

#failed codes
#sequence build status
Fbuildsequ=1
Fdoexcl=2
#run process status
Ftimecorr=3
Ffillraw=4
Fsinope=5
Ffillsinope=6
Fresetexcl=7
#sequence process status
Fwritesequfile=8
Ffilesavail=9
Fnoccfile=10
Fnocacofile=11
Fmerppcc=12
Fmerppcaco=13
Fcallisto=14
Ffillcalib=15
Ffillsignal=16
Fstar=17
Ffillstar=18
#dataset process status
Fwritedatasetfile=19
Fstardone=20
Fganymed=21
Ffillganymed=22
#again run process status
FCompmux=26
Fdowebplots=27
#again mc run process status
Fmccallisto=28
Ffillmccalib=29
Ffillmcsignal=30
Fmcstar=31
Ffillmcstar=32
Fcorsikasimtel=33
Fchimp=34
Fchimpcp=35
Fctastar=36
Fctastarcp=37
FctastereoA=38
FctastereoB=39
FctastereoC=40
FctastereoD=41
FctastereoE=42
FctastereoF=43
FctastereoG=44
FctastereoH=45
Fctastereocp=46

# setup for jobmanager:
#   log files (can't be defined in script itself, as script can run longer 
#   than one day
jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
jmscriptlog=$runlogpath/jobmanager`date +%F`.log

# check if rc-files are available
if ! ls $steps >/dev/null
then
   echo "Can't find steps.rc ($steps)"
   finish
fi
if ! ls $sqlrc >/dev/null
then
   echo "Can't find sql.rc ($sqlrc)"
   finish
fi

# resetting values
pno=0
totalpno=0
running=0
queued=0
runningscript=0
queuedscript=0
stillinqueue=0


# alias (we cannot check the beginning of the line due to
# color codes in filldotraw.C)
alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'


# in the following the functions, which are needed by several scripts, are 
# defined

# function to check if a process is already locked
#  command line option can be used to execute something, e.g. 'continue'
function checklock()
{
   if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
   then 
      printprocesslog "WARN lockfile $lockfile exists"
      $@
      exit
   else
      printprocesslog "INFO created lockfile $lockfile"
   fi
}

# print the current status values
function printstatusvalues()
{
   echo "the current values are:"
   echo " starttime=$starttime"
   echo " stoptime=$stoptime"
   echo " returncode=$returncode"
   echo "-- check: -$check-"
   echo ""
}

# get the db-setup from the sql.rc
function getdbsetup()
{
   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
   pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
#   echo "setup: "
#   echo " db: "$db
#   echo " pw: "$pw
#   echo " us: "$us
#   echo " ho: "$ho
}

# function to get the needed information from the dependencies-file steps.rc
function getstepinfo()
{
   getdbsetup
   needs=`grep "$step[.]Needs:" $steps | grep -v '#' | sed -e "s/$step[.]Needs://"`
   noderestricted=`grep "$step[.]NodeRestricted:" $steps | grep -v '#' | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
   prims=( `grep "$step[.]Primaries:" $steps | grep -v '#' | sed -e "s/$step[.]Primaries://"` )
#   echo " needs: $needs"
#   echo " noderestricted: $noderestricted"
#   echo " prims: ${prims[@]}"
}

# function to get the primaries of a step from the dependencies-file steps.rc
function getprimary()
{
   getdbsetup
   grep $@"[.]Primaries:" $steps | grep -v '#' | sed -e "s/$@[.]Primaries://"
}

# function to get the join of a step 
#  (normal join but using different primaries)
#  from the dependencies-file steps.rc
function getjoin()
{
   getdbsetup
   grep $@"[.]Join:" $steps | grep -v '#' | sed -e "s/$@[.]Join://"
}

# function to get the special join of a step 
#  (i.e. join with a different table) 
#  from the dependencies-file steps.rc
function getspecialjoin()
{
   getdbsetup
   grep $@"[.]SpecialJoin:" $steps | grep -v '#' | sed -e "s/$@[.]SpecialJoin://"
}

# function to get the needs of a step from the dependencies-file steps.rc
function getneeds()
{
   getdbsetup
   grep $@"[.]Needs:" $steps | sed -e "s/$@[.]Needs://"
}

# function to create the middle part of a query
#  which is identical for the functions getstatus() and gettodo()
function middlepartofquery()
{
   # add from which table the information is queried
   query=$query" FROM "$step"Status "
   # add the joins to the tables in which the status of the preceding steps is stored
   for need in $needs
   do
      needprims=( `getprimary $need` )
      # make sure that the correct joins are used
      # in case the primaries of the step and the need are 
      #   the same, the tables are join with these primaries
      # otherwise it is checked whether there has to be a 
      #   join with different primaries (indicated as 'Join'
      #   in the steps.rc
      # 'SpecialJoins' do not have to be taken into account here
      if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
      then 
         query=$query" LEFT JOIN "$need"Status USING (${prims[@]}) "
      else 
         stdjoin=`getjoin $need`
         if ! [ "$stdjoin" = "" ]
         then
            query=$query" "$stdjoin" "
         fi
      fi
   done
   # add condition
   query=$query" WHERE "
   # add condition for the status of the peceding steps
   counter=0
   for need in $needs
   do
      if [ $counter -gt 0 ]
      then
         query=$query" AND "
      fi
      needprims=( `getprimary $need` )
      # in case the primaries of the tables agree
      #   or there is just a normal 'Join' needed
      #   only the condition is added to the query
      # for tables which need a join with a different
      #   table, a special query is added using the
      #   'SpecialJoin' from the steps.rc
      if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ] || ! [ "$stdjoin" = "" ]
      then 
         query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
         query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
         query=$query" ISNULL("$need"Status.fReturnCode) "
      else 
         query=$query" (SELECT COUNT(*) FROM "$need"Status "
         query=$query" "`getspecialjoin $need`" "
         query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
         for (( j=1 ; j < ${#prims[@]} ; j++ ))
         do
            query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
         done
         query=$query") = (SELECT COUNT(*) FROM "$need"Status "
         query=$query" "`getspecialjoin $need`" "
         query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
         for (( j=1 ; j < ${#prims[@]} ; j++ ))
         do
            query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
         done
         query=$query" AND NOT ISNULL(fStartTime) "
         query=$query" AND NOT ISNULL(fStopTime) "
         query=$query" AND ISNULL(fReturnCode)) "
      fi
      counter=`echo $counter + 1 | bc -l`
   done
   # add condition for the status of the step itself
   if [ $counter -gt 0 ]
   then
      query=$query" AND "
   fi
   query=$query" ISNULL("$step"Status.fStartTime) "
   query=$query" AND ISNULL("$step"Status.fStopTime) "
   query=$query" AND ISNULL("$step"Status.fReturnCode) "
}

# function to get todolist
#   returns the next or the list of next steps
function gettodo()
{
   # reset the variable for the number of the next step 
   process=
   printprocesslog "INFO getting todo..."
   getstepinfo
   # get query
   query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
   for (( j=1 ; j < ${#prims[@]} ; j++ ))
   do
      query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
   done
   # get middle par of query 
   middlepartofquery
   # add requirement for production host in case it is needed
   if [ "$2 " != " " ]
   then 
      query=$query" AND fProductionHostKEY=$2 "
   fi
   # order by priority to the the number of the next step to be done
   query=$query" ORDER BY "$step"Status.fPriority desc "
   # add limitation in case only one or a limited number of 
   #  processes should be executed
   if [ "$1 " != " " ]
   then 
      query=$query" limit 0, $1 "
   fi
   # print query 
   #echo " gettodo QUERY: "$query
   printprocesslog "INFO gettodo QUERY: "$query
   # execute query 
   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
      finish
   fi
   # get numbers of next step from mysql result
   if [ "$process" = "" ]
   then
      printprocesslog "INFO => nothing to do"
      finish
   else
      primaries=( $process )
      num=`expr ${#primaries[@]} / ${#prims[@]} `
   fi
}

# function to get the number of processes which still have to be done
function getstatus()
{
   # reset the variable for the number of steps to be done
   numproc=
   getstepinfo
   # get query
   query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
   # get middle part of query 
   middlepartofquery
   # add requirement for production host in case it is needed
   if [ "$1 " != " " ]
   then 
      query=$query" AND fProductionHostKEY=$1 "
   fi
   # group by an 'artifical' column to get the number of lines
   query=$query" GROUP BY 2 "
   # printing query
   #echo " getstatus QUERY: "$query
   printprocesslog "INFO getstatus QUERY: "$query
   # execute query
   if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
   then
      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
      echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
      continue
   fi
   # get number of processes from mysql result
   if [ "${numprocs[1]}" = "" ]
   then
      numproc=0
   else
      numproc=${numprocs[0]}
   fi
}

# function to set status of a process in the db
function setstatus()
{
   # remark:
   # this function does not include the 'Default' flag 
   # for resetting steps

   # for dowebplots (there are steps which have no entry in the DB)
   if [ "$step" = "no" ]
   then
      return
   fi
   
   # reset status values
   starttime=NULL
   stoptime=NULL
   returncode=NULL
   # evaluate the status values
   case $@ in
      start)   printprocesslog "INFO setstatus start"
               starttime="Now()"
               ;;
       stop)   case $check in
                  ok)  printprocesslog "INFO setstatus stop - ok"
                       starttime=noreset
                       stoptime="Now()"
                       ;;
                  no)  printprocesslog "INFO setstatus stop - nothing new"
                       check="ok"
                       ;;
                   *)  printprocesslog "INFO setstatus stop - failed"
                       starttime=noreset
                       stoptime="Now()"
                       if [ "$check" == "" ]
                       then
                          returncode=1
                       else
                          returncode=$check
                       fi
                       check="ok"
                       ;;
               esac
               ;;
          *)   printprocesslog "ERROR function setstatus got wrong variable"
               finish
               ;;
   esac
   # get 
   getstepinfo
   
   # get the influences from the steps.rc by evaluating the needs of all steps
   influences=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
   
   # get query
   query=" UPDATE "$step"Status "
   # add joins to the influenced tables
   stepspecialjoin=""
   for influence in $influences
   do
      inflprims=( `getprimary $influence` )
      # make sure that the correct joins are used
      # in case the primaries of the step and the influence are 
      #   the same, the tables are join with these primaries
      # otherwise they are joined with the primaries of the influence
      if [ "`echo ${inflprims[@]}`" == "`echo ${prims[@]}`" ]
      then 
         query=$query" LEFT JOIN "$influence"Status USING (${prims[@]}) "
      else 
         prims2=`getprimary $step | sed -e "s/\ //g"`
         primstest=`echo ${inflprims[@]} | sed -e "s/\ //g" | sed -e "s/$prims2//g"`
         if [ "$primstest" == "`echo ${inflprims[@]} | sed -e "s/\ //g"`" ]
         then
            if [ "$stepspecialjoin" == "" ]
            then
               stepspecialjoin=`getspecialjoin $step`
               query=$query" "$stepspecialjoin" "
            fi
            query=$query" LEFT JOIN "$influence"Status USING (${inflprims[@]}) "
         else
            infljoin=`getjoin $influence`
            if [ "$infljoin" = "" ]
            then
               query=$query" LEFT JOIN "$influence"Status USING (${prims[@]}) "
            else
               query=$query" "$infljoin
               # warning: this is a workaround
               #  it has to be checked whether it works for all steps
               stepspecialjoin="not empty"
            fi
         fi
      fi
   done
   # set the status values according to the new status of the step
   query=$query" SET "
   if ! [ "$starttime" = "noreset" ]
   then
      query=$query" "$step"Status.fStartTime=$starttime, "
   fi
   query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
   # set also the status values of the influenced steps
   for influence in $influences
   do
      query=$query", "$influence"Status.fStartTime=NULL "
      query=$query", "$influence"Status.fStopTime=NULL "
      query=$query", "$influence"Status.fReturnCode=NULL "
   done
   # give the condition for which step the status values have to be set
   query=$query" WHERE "
   if [ "$s" = "" ]
   then
      s=0
   fi
   query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
   for (( j=1 ; j < ${#prims[@]} ; j++ ))
   do
      query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
   done   
   # print query
   #echo " setstatus QUERY: "$query
   printprocesslog "INFO setstatus QUERY: "$query
   # execute query
   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
   then
      printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
      finish
   fi
}

# function to send a mysql query
function sendquery()
{
   getdbsetup
   printprocesslog "INFO sendquery QUERY: "$query
   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   then
      printprocesslog "ERROR could not query db (program: $program, function sendquery)"
      return 1
   fi
   if [ "$val" = "NULL" ]
   then
      val=
   fi
   echo $val
   return 0
}