#!/bin/bash

# to treat aliases in bash-script correctly
shopt -s expand_aliases

# check if script has been started with absolute path
if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
then 
   echo "Please start your script with an absolute path."
   exit
fi

if [ "$AUTOMATIONSETUP" = "" ]
then 
   echo "Please set the environment variable \$AUTOMATIONSETUP."
   exit
fi

if [ "$SOURCEFILEPATH" = "" ]
then 
   export SOURCEFILEPATH=`dirname $0`
fi
if [ "$SCRIPTNAME" = "" ]
then 
   SCRIPTNAME=`basename $0`
fi

source $SOURCEFILEPATH/../Setup/setup.$AUTOMATIONSETUP

if [ "$mars" = "" ]
then 
   echo "Please set the path for MARS."
   exit
fi

datetime=`date +%F-%H-%M-%S`

if [ "$transferdelay" = "" ]
then 
   # setup where immediate report of problem is needed, i.e. mainly LP
   checknight=`date +%Y%m%d --date="-19HOUR"`
else
   # any setup where transfer might cause a delay in arrival of files
   checknight=`date +%Y%m%d  --date="-${transferdelay}day"`
fi

# function to make sure that a directory is made
function makedir()
{
   if [ ! -d $@ ]
   then
      if [ "$processlog" = "" ] || [ "$logfile" = "" ]
      then
         mkdir -p $@ 
      else
         mkdir -pv $@ 
      fi
      if [ ! -d $@ ]
      then 
         if ! [ "$processlog" = "" ]
         then
            echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
         else
            echo "could not make dir "$@
         fi
         if ls $lockfile >/dev/null 2>&1
         then 
            rm -v $lockfile
         fi
         exit
      fi
   fi
}

# logging paths for runlogs and processlog
runlogpath=$logpath/run/`date +%Y/%m/%d`
processlogpath=$logpath/processlog
makedir $runlogpath
makedir $processlogpath
processlog=$processlogpath/process`date +%F`.log

makedir $lockpath


# function to provide proper logging in a single logfile ($processlog)
function printprocesslog
{
   makedir $processlogpath
   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
}

# function to exit a script properly
function finish()
{
   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
   then 
      printprocesslog "DEBUG " `rm -v $lockfile`
   fi
   printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
   # fixme: handle different cases - sometimes exit of script is needed, sometimes only continue
   exit
}


# set checkvalue to ok at the beginning of the scripts
check="ok"

# setup for jobmanager:
#   log files (can't be defined in script itself, as script can run longer 
#   than one day
jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log

# check if rc-files are available
if ! ls $steps >/dev/null
then
   echo "Can't find steps.rc ($steps)"
   finish
fi
if ! ls $sqlrc >/dev/null 2>&1 || [ "$sqlrc" = "" ]
then
   echo "Can't find sql.rc ($sqlrc)"
   finish
fi

# resetting values for jobmanager
pno=0
totalpno=0
running=0
queued=0
runningscript=0
queuedscript=0
stillinqueue=0


# alias (we cannot check the beginning of the line due to
# color codes in filldotraw.C)
alias 'intgrep'='grep -E -o \\\("(int|Bool_t)"\\\)[0-9]+$ | grep -E -o [0-9]+'


# in the following the functions, which are needed by several scripts, are 
# defined

# function to check if a process is already locked
#  command line option can be used to execute something, e.g. 'continue'
function checklock()
{
   if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
   then 
      if find $lockfile -amin -5
      then
         printprocesslog "INFO lockfile $lockfile exists"
      else
         printprocesslog "WARN lockfile $lockfile exists"
      fi
      $@
      exit
   else
      printprocesslog "DEBUG created lockfile $lockfile"
   fi
}

# print the current status values
function printstatusvalues()
{
   echo "the current values are:"
   echo " starttime=$starttime"
   echo " stoptime=$stoptime"
   echo " availtime=$availtime"
   echo " returncode=$returncode"
   echo "-- check: -$check-"
   echo ""
}

# get the db-setup from the sql.rc
function getdbsetup()
{
   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
   pw="--password="`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
   if [ "$sqlpw" != "" ]
   then 
      sqlpwfile="--defaults-file=$sqlpw"
      pw=""
   fi
#   echo "setup: "
#   echo " db: "$db
#   echo " pw: "$pw
#   echo " us: "$us
#   echo " ho: "$ho
}

# function to send a mysql query
function sendquery()
{
   getdbsetup
   printprocesslog "DEBUG sendquery QUERY: "$query
   val=`mysql $sqlpwfile -s -u $us $pw --host=$ho $db -e " $query " 2>&1`
   checkmysql=`echo $?`
   if [ $checkmysql -gt 0 ]
   then
      printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us" (program: $program, function sendquery) "
      printprocesslog "MYSQLERROR [returncode: "$checkmysql"] "$val
      error=`echo $val | grep -E -o 'ERROR [1-9]{1,4}' | grep -E -o '[1-9]{1,4}'`
      # here possible reaction to mysql error
      if [ $error -eq 1213 ]
      then 
         printprocesslog "WARN Deadlock found. Should resend query."
         printprocesslog "DEBUG sendquery QUERY: [2nd try] "$query
         val=`mysql $sqlpwfile -s -u $us $pw --host=$ho $db -e " $query " 2>&1`
         checkmysql2=`echo $?`
         if [ $checkmysql2 -gt 0 ]
         then
            printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us" (program: $program, function sendquery) [2nd try]"
            printprocesslog "MYSQLERROR [returncode: "$checkmysql"] "$val" [2nd try]"
            error2=`echo $val | grep -E -o 'ERROR [1-9]{1,4}' | grep -E -o '[1-9]{1,4}'`
            val=
            finish
         else
            if [ "$val" = "NULL" ]
            then
               val=
            fi
            echo $val
            return 0
         fi
      fi
      val=
      finish
   else
      warning=`echo $val | grep -o -E 'mysql: \[Warning\]'`
      if [ "$warning" != "" ]
      then
         printprocesslog "WARN possible problem to query DB "$db" on host "$ho" with user "$us" (program: $program, function sendquery) "
         printprocesslog "MYSQLWARN [returncode: "$checkmysql"] "$val
         val=
         finish
      fi
   fi
   if [ "$val" = "NULL" ]
   then
      val=
   fi
   echo $val
   return 0
}

# function to get information from the setupfile $steps
function getfromsetup()
{
   grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
}

# function to get the needed information from the dependencies-file steps.rc
function getstepinfo()
{
   getdbsetup
   needs=( `getfromsetup $step "Needs"` )
   noderestricted=`getfromsetup $step "NodeRestricted"`
   prims=( `getfromsetup $step "Primaries"` )
   maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
   sort=`getfromsetup $step "SortDirection" | sed -e "s/\ //g"`
#   echo " maintable: "$maintable
#   echo " needs: "${needs[@]}
#   echo " noderestricted: "$noderestricted
#   echo " prims: "${prims[@]}
}

# function to get the joins needed for the get/set status queries
function getalljoins()
{
   # add table
   query=$query" "$maintable"Status"
   # add special join
   query=$query" "`getfromsetup $maintable "SpecialJoin"`
   # add join for step unless step is the same as maintable
   if ! [ "$step" = "$maintable" ]
   then 
      query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
   fi
   # add joins for influences or needs
   for otherstep in ${othersteps[@]}
   do
      if ! [ "$otherstep" = "$maintable" ]
      then 
         query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
      fi
   done
}

# function to create the middle part of a query
#  which is identical for the functions getstatus() and gettodo()
function getstatusquery()
{
   # add from which table the information is queried
   query=$query" FROM "
   othersteps=${needs[@]}
   getalljoins
   # add condition
   query=$query" WHERE "
   # add condition for step, i.e. step is not yet done
   query=$query" ISNULL("$step"Status.fStartTime) "
   query=$query" AND ISNULL("$step"Status.fStopTime) "
   query=$query" AND ISNULL("$step"Status.fAvailable) "
   query=$query" AND ISNULL("$step"Status.fReturnCode) "
   # add requirement for production host in case it is needed
   if [ "$1 " != " " ]
   then 
      query=$query" AND fProductionHostKEY=$2 "
   fi
   query=$query`getfromsetup $step "SpecialWhere"`
   # add condition for needs, i.e. that step is done
   for (( k=0 ; k < ${#needs[@]} ; k++ ))
   do
#      if [ $k -eq 0 ]
#      then 
#         query=$query" HAVING "
#      else
         query=$query" AND "
#      fi
#      query=$query" COUNT(*)=COUNT(IF("
      query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
      query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
      query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
      query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
#      query=$query" , 1, NULL)) "
   done
#   if ! echo $query | grep UPDATE >/dev/null 2>&1
#   then 
#      query=$query" GROUP BY "${prims[@]}
#   fi
}

# function to get todolist
#   returns the next or the list of next steps
function gettodo()
{
   # reset the variable for the number of the next step 
   process=
   printprocesslog "DEBUG getting todo for step $step..."
   getstepinfo
   # get query
   query=" SELECT "${prims[@]}
   getstatusquery $2
   # order by priority to the the number of the next step to be done
   query=$query" ORDER BY "$step"Status.fPriority "
   if [ "$sort" = "" ]
   then 
      query=$query" DESC "
   else
      query=$query" "$sort
   fi
   # add limitation in case only one or a limited number of 
   #  processes should be executed
   if [ "$1 " != " " ]
   then 
      query=$query" limit 0, $1 "
   fi
   # print query 
   #printprocesslog "DEBUG gettodo for step $step QUERY: "$query
   # execute query 
   process=`sendquery`
   #if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
   #then
   #   printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
   #   finish
   #fi
   # get numbers of next step from mysql result
   if [ "$process" = "" ]
   then
      printprocesslog "DEBUG => nothing to do"
      finish
   else
      primaries=( $process )
      num=`expr ${#primaries[@]} / ${#prims[@]} `
   fi
}

# function to get the number of processes which still have to be done
function getstatus()
{
   printprocesslog "DEBUG getstatus for step "$step
   # reset the variable for the number of steps to be done
   numproc=0
   getstepinfo
   # get query
   query=" SELECT "${prims[@]}
   getstatusquery $1
   # print query
   #printprocesslog "DEBUG getstatus for step $step QUERY: "$query
   # execute query
   #numproc=`sendquery `#cannot be done with sendquery, because of row counting
   if ! numproc=`mysql $sqlpwfile -s -u $us $pw --host=$ho $db -e " $query " | wc -l`
   then
      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
      echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
      continue
   fi
}

# function to set status of a process in the db
function setstatus()
{
   # remark:
   # this function does not include the 'Default' flag 
   # for resetting steps

   # for dowebplots (there are steps which have no entry in the DB)
   if [ "$step" = "no" ]
   then
      return
   fi
   
   printprocesslog "DEBUG setstatus for step "$step

   # reset status values
   starttime=NULL
   stoptime=NULL
   availtime=NULL
   returncode=NULL
   # evaluate the status values
   case $@ in
      start)   printprocesslog "DEBUG setstatus start"
               starttime="Now()"
               ;;
      startj)  printprocesslog "DEBUG setstatus startj"
               starttime="'1971-01-01 01:01:01'" # special recognizable datetime, i.e. "reserved by JobManager for processing"
               ;;
       stop)   case $check in
                  ok)  printprocesslog "DEBUB setstatus stop - ok"
                       starttime=noreset
                       stoptime="Now()"
                       if [ "$processingsite" = "$storagesite" ]
                       then
                          availtime="Now()"
                       fi
                       ;;
                  no)  printprocesslog "DEBUG setstatus stop - nothing new"
                       check="ok"
                       ;;
                   *)  printprocesslog "DEBUG setstatus stop - failed"
                       starttime=noreset
                       stoptime="Now()"
                       if [ "$processingsite" = "$storagesite" ]
                       then
                          availtime="Now()"
                       fi
                       if [ "$check" == "" ]
                       then
                          returncode=1
                       else
                          returncode=$check
                       fi
                       check="ok"
                       ;;
               esac
               ;;
          *)   printprocesslog "ERROR function setstatus got wrong variable"
               finish
               ;;
   esac
   
   # get 
   getstepinfo
   
   # get the influences from the steps.rc by evaluating the needs of all steps
   othersteps=`grep $step $steps | grep -v '#' | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
   
   # get query
   query=" UPDATE "
   getalljoins
   # set the status values according to the new status of the step
   query=$query" SET "
   if ! [ "$starttime" = "noreset" ]
   then
      query=$query" "$step"Status.fStartTime=$starttime, "
   fi
   query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
   query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
   # set also the status values of the influenced steps
   for otherstep in $othersteps
   do
      query=$query", "$otherstep"Status.fStartTime=NULL "
      query=$query", "$otherstep"Status.fStopTime=NULL "
      query=$query", "$otherstep"Status.fAvailable=NULL "
      query=$query", "$otherstep"Status.fReturnCode=NULL "
      query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
   done
   # give the condition for which step the status values have to be set
   query=$query" WHERE "
   if [ "$s" = "" ]
   then
      s=0
   fi
   query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
   for (( j=1 ; j < ${#prims[@]} ; j++ ))
   do
      query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
   done 
   # add additional query to allow for locking in db
   if [ "$1" = "start" ]
   then 
      # process job only if it is not yet being processed
      #  i.e. either StartTime NULL or '1971-01-01 01:01:01' (reserved by JobManager for processing)
      query=$query" AND (ISNULL("$step"Status.fStartTime) OR "$step"Status.fStartTime='1971-01-01 01:01:01')"
   fi
   # add row count to know how many rows have been changed
   query=$query"; SELECT ROW_COUNT();"
   # print query
   #printprocesslog "DEBUG setstatus for step $step QUERY: "$query
   #echo "DEBUG setstatus for step $step QUERY: "$query
   # execute query
   numchanged=`sendquery`
   # should not be needed anymore once finish does proper exit
   printprocesslog "DEBUG numchanged "$numchanged
   if [ "$numchanged" = "" ]
   then
      printprocesslog "DEBUG numchanged empty."
      finish
   fi
   if [ $numchanged -gt 0 ]
   then
      printprocesslog "INFO successful set of status in DB."
      #echo "INFO successful set of status in DB."
   else
      # action may be taken in script using $numchanged
      printprocesslog "DEBUG status in DB was already set by another process "
      #echo "ERROR status in DB was already set by another process "
   fi
}

function getdates()
{
   case $1 in
      # all dates
      all) 
         printprocesslog "DEBUG getdates case 'all'"
         dates=( `find $auxdata -mindepth 3 -type d | sort -r | sed "s/\${auxdata_for_sed}//g" | sed -e 's/^\///'` ) 
         ;;
      # certain date
      [0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]|[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]-[0-9][0-9][0-9])
         d=`echo $1 | grep -o '[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]'`
         range=`echo $1 | grep -o '[0-9][0-9][0-9]$'`
         range=`echo $range | sed -e 's/^0//' -e 's/^0//'`
         printprocesslog "DEBUG getdates - certain date "$d"."
         dates=( $d )
         if [ "$range" != "" ]
         then 
            printprocesslog "DEBUG getdates - add the last "$range" days."
            yy=`echo $d | cut -c 1-4`
            mm=`echo $d | cut -c 6-7`
            dd=`echo $d | cut -c 9-10`
            for (( numdates=1 ; numdates <= $range ; numdates++ ))
            do 
               numhours=`echo " 12 + ( $numdates - 1 ) * 24 " | bc -l`
               dates=( ${dates[@]} `date +%Y/%m/%d --date=$yy/$mm/$dd"-"$numhours"hour"` )
            done
         fi
         ;;
      # certain number of dates (between 0 and 9999)
      [1-9][0-9][0-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9]|[1-9])
         # get last n nights
         printprocesslog "DEBUG getdates - get the last "$1" days."
         for (( numdates=1 ; numdates <= $1 ; numdates++ ))
         do 
            numhours=`echo " 12 + ( $numdates - 1 ) * 24 " | bc -l`
            dates=( ${dates[@]} `date +%Y/%m/%d --date="-"$numhours"hour"` )
         done
         # hour-dependent number of dates
         if [ "$2" != "" ] && [ "$3" != "" ]
         then 
            # get current hour
            hour=`date +%k`
            if [ $hour -le $2 ] || [ $hour -ge $3 ]
            then
               printprocesslog "DEBUG getdates - get the current night."
               dates=( `date +%Y/%m/%d --date="-19hour"` )
            fi
         fi
         ;;
      # certain number of dates in the future (between 0 and 9)
      +[1-9])
         # get next n nights
         printprocesslog "DEBUG getdates - get the next "$1" days (incl today)."
         for (( numdates=1 ; numdates <= $1 ; numdates++ ))
         do 
            numhours=`echo " 0 + ( $numdates - 1 ) * 24 " | bc -l`
            dates=( ${dates[@]} `date +%Y/%m/%d --date="+"$numhours"hour"` )
         done
         # hour-dependent number of dates
         if [ "$2" != "" ] && [ "$3" != "" ]
         then 
            # get current hour
            hour=`date +%k`
            if [ $hour -le $2 ] || [ $hour -ge $3 ]
            then
               printprocesslog "DEBUG getdates - get the current night."
               dates=( `date +%Y/%m/%d --date="-12hour"` )
            fi
         fi
         ;;
      *) # nothing valid given
         echo "Please give valid option (YYYY/MM/DD[-RRR] or 1-9999 or +0-9)"
         finish
         ;;
   esac
}

# missing: implement usage of this function in Fill*.sh
function check_file_avail()
{
   if [ "$1" = "" ]
   then
      echo "Please give file name as first argument."
      finish
   fi
   
   printprocesslog "INFO checking availability of "$1
   if ! [ -e $1 ]
   then 
      filenight=`basename $1 | cut -c 1-8`
      # treat raw files differently than auxfiles
      checkraw=`echo $1 | grep -E -o raw`
      if [ "$checkraw" = "raw" ]
      then 
         if [ $filenight -le $checknight ] 
         then 
            printprocesslog "WARN "$1" not found."
         else
            printprocesslog "INFO "$1" not found."
         fi
         return 1
      fi
      # only for aux files check of DB is needed
      # treat ratescan files 
      checkratescan=`echo $1 | grep -E -o RATE_SCAN_DATA`
      if [ "$checkratescan" = "RATE_SCAN_DATA" ] &&  [ $filenight -gt 20130408 ]
      then
         #query="SELECT fMeasurementTypeKey from MeasurementType WHERE fMeasurementTypeName like 'Ratescan%'"
         #ratescantypes=`sendquery`
         query="SELECT COUNT(*) FROM Schedule WHERE fStart BETWEEN "$scheduletart" AND "$schedulestop" AND fMeasurementTypeKey IN (5,8) "
         numratescans=`sendquery`
         if [ $numratescans -gt 0 ]
         then 
            printprocesslog "ERROR "$1" not found."
         else
            printprocesslog "INFO "$1" not found."
         fi
         return 1
      fi
      query="SELECT COUNT(*) FROM RunInfo WHERE fNight="$filenight" AND fRunTypeKey=1"
      numdatruns=`sendquery`
      # for some files it is more severe when they are missing
      checkfile=`echo $1 | grep -E -o 'DRIVE'\|'RATES'`
      # print INFO/WARN/ERROR depending on severity
      if [ $numdatruns -eq 0 ]
      then 
         # in case no data runs are available from this night, it's not a problem if files are missing
         printprocesslog "INFO "$1" not found."
      else
         if [ "$checkfile" != "" ]
         then
            # drive and trigger-rates files are needed for analysis
            #printprocesslog "ERROR "$1" not found."
            if [ $filenight -le $checknight ] 
            then 
               printprocesslog "ERROR "$1" not found."
            else
               printprocesslog "WARN "$1" not found."
            fi
         else
            # other aux-files would be good to have
            #printprocesslog "WARN "$1" not found."
            if [ $filenight -le $checknight ] 
            then 
               printprocesslog "WARN "$1" not found."
            else
               printprocesslog "INFO "$1" not found."
            fi
         fi
      fi
      return 1
   fi
   
   # file available 
   return 0
}