Index: trunk/DataCheck/Sourcefile.sh
===================================================================
--- trunk/DataCheck/Sourcefile.sh	(revision 13035)
+++ trunk/DataCheck/Sourcefile.sh	(revision 13036)
@@ -2,16 +2,10 @@
 
 # data paths
-auxdata=/loc_data/aux
-rawdata=/loc_data/raw
+auxdata=/data00/fact-construction/aux
+rawdata=/data00/fact-construction/raw
 rawdata_for_sed=$(printf "%s\n" "$rawdata" | sed 's/[][\.*^$(){}?+|/]/\\&/g')
 ziprawdata=/loc_data/zipraw
 ziprawdata_for_sed=$(printf "%s\n" "$ziprawdata" | sed 's/[][\.*^$(){}?+|/]/\\&/g')
 
-# path to FACT++ version
-factpath=/home/fact/SW.automatic.processing/FACT++.2012.01.25
-
-# path to MARS CheOs version
-mars=/home/fact/SW.automatic.processing/Mars.CheObs.svn.2012.01.25
-
 # setup to use ftools
 export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
@@ -19,10 +13,468 @@
 
 # software versions
-export factpath=/home/fact/SW.automatic.processing/FACT++.2012.01.25
-export mars=/home/fact/SW.automatic.processing/Mars.CheObs.svn.2012.01.25
-
-export AUTOMATIONSETUP="lp.fact"
-export SOURCEFILEPATH=$mars/datacenter/scripts
-
-source $SOURCEFILEPATH/sourcefile
-
+#export factpath=/home/fact/SW.automatic.processing/FACT++.2012.01.25
+export mars=~/Mars.von.Thomas.2012.03.07
+
+export AUTOMATIONSETUP="isdc.fact"
+#export SOURCEFILEPATH=`dirname/datacenter/scripts
+
+#source $SOURCEFILEPATH/sourcefile
+
+#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
+#
+#   Copyright: MAGIC Software Development, 2000-2010
+#
+#
+# ========================================================================
+#
+# This a resource file for the scripts, in which the standard paths and 
+# functions, which are needed more often are stored. 
+# Only constant variables are stored here, changing variables are stored 
+# in datacenter/scripts/setup
+#
+
+# to treat aliases in bash-script correctly
+shopt -s expand_aliases
+
+# check if script has been started with absolute path
+if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
+then 
+   echo "Please start your script with an absolute path."
+   exit
+fi
+
+if [ "$AUTOMATIONSETUP" = "" ]
+then 
+   echo "Please set the environment variable \$AUTOMATIONSETUP."
+   exit
+fi
+
+if [ "$SOURCEFILEPATH" = "" ]
+then 
+   export SOURCEFILEPATH=`dirname $0`
+fi
+if [ "$SCRIPTNAME" = "" ]
+then 
+   SCRIPTNAME=`basename $0`
+fi
+
+source $SOURCEFILEPATH/../Setup/setup.$AUTOMATIONSETUP
+
+datetime=`date +%F-%H-%M-%S`
+
+
+# function to make sure that a directory is made
+function makedir()
+{
+   if [ ! -d $@ ]
+   then
+      mkdir -pv $@ 
+      if [ ! -d $@ ]
+      then 
+         if ! [ "$processlog" = "" ]
+         then
+            echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
+         else
+            echo "could not make dir "$@
+         fi
+         if ls $lockfile >/dev/null 2>&1
+         then 
+            rm -v $lockfile
+         fi
+         exit
+      fi
+   fi
+}
+
+# logging paths for runlogs and processlog
+runlogpath=$logpath/run/`date +%Y/%m/%d`
+processlogpath=$logpath/processlog
+makedir $runlogpath
+makedir $processlogpath
+processlog=$processlogpath/process`date +%F`.log
+
+makedir $lockpath
+
+
+# function to provide proper logging in a single logfile ($processlog)
+function printprocesslog
+{
+   makedir $processlogpath
+   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
+}
+
+# function to exit a script properly
+function finish()
+{
+   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
+   then 
+      printprocesslog "DEBUG " `rm -v $lockfile`
+   fi
+   printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
+   exit
+}
+
+
+# set checkvalue to ok at the beginning of the scripts
+check="ok"
+
+# setup for jobmanager:
+#   log files (can't be defined in script itself, as script can run longer 
+#   than one day
+jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
+jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
+
+# check if rc-files are available
+if ! ls $steps >/dev/null
+then
+   echo "Can't find steps.rc ($steps)"
+   finish
+fi
+if ! ls $sqlrc >/dev/null
+then
+   echo "Can't find sql.rc ($sqlrc)"
+   finish
+fi
+
+# resetting values for jobmanager
+pno=0
+totalpno=0
+running=0
+queued=0
+runningscript=0
+queuedscript=0
+stillinqueue=0
+
+
+# alias (we cannot check the beginning of the line due to
+# color codes in filldotraw.C)
+alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
+
+
+# in the following the functions, which are needed by several scripts, are 
+# defined
+
+# function to check if a process is already locked
+#  command line option can be used to execute something, e.g. 'continue'
+function checklock()
+{
+   if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
+   then 
+      printprocesslog "WARN lockfile $lockfile exists"
+      $@
+      exit
+   else
+      printprocesslog "DEBUG created lockfile $lockfile"
+   fi
+}
+
+# print the current status values
+function printstatusvalues()
+{
+   echo "the current values are:"
+   echo " starttime=$starttime"
+   echo " stoptime=$stoptime"
+   echo " availtime=$availtime"
+   echo " returncode=$returncode"
+   echo "-- check: -$check-"
+   echo ""
+}
+
+# get the db-setup from the sql.rc
+function getdbsetup()
+{
+   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
+   pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
+   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
+   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
+#   echo "setup: "
+#   echo " db: "$db
+#   echo " pw: "$pw
+#   echo " us: "$us
+#   echo " ho: "$ho
+}
+
+# function to send a mysql query
+function sendquery()
+{
+   getdbsetup
+   printprocesslog "DEBUG sendquery QUERY: "$query
+   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
+   then
+      printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us
+      #printprocesslog "ERROR could not query db (program: $program, function sendquery)"
+      #return 1 #why???
+      finish
+   fi
+   if [ "$val" = "NULL" ]
+   then
+      val=
+   fi
+   echo $val
+   return 0
+}
+
+# function to get information from the setupfile $steps
+function getfromsetup()
+{
+   grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
+}
+
+# function to get the needed information from the dependencies-file steps.rc
+function getstepinfo()
+{
+   getdbsetup
+   needs=( `getfromsetup $step "Needs"` )
+   noderestricted=`getfromsetup $step "NodeRestricted"`
+   prims=( `getfromsetup $step "Primaries"` )
+   maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
+#   echo " maintable: "$maintable
+#   echo " needs: "${needs[@]}
+#   echo " noderestricted: "$noderestricted
+#   echo " prims: "${prims[@]}
+}
+
+# function to get the joins needed for the get/set status queries
+function getalljoins()
+{
+   # add table
+   query=$query" "$maintable"Status"
+   # add special join
+   query=$query" "`getfromsetup $maintable "SpecialJoin"`
+   # add join for step unless step is the same as maintable
+   if ! [ "$step" = "$maintable" ]
+   then 
+      query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
+   fi
+   # add joins for influences or needs
+   for otherstep in ${othersteps[@]}
+   do
+      if ! [ "$otherstep" = "$maintable" ]
+      then 
+         query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
+      fi
+   done
+}
+
+# function to create the middle part of a query
+#  which is identical for the functions getstatus() and gettodo()
+function getstatusquery()
+{
+   # add from which table the information is queried
+   query=$query" FROM "
+   othersteps=${needs[@]}
+   getalljoins
+   # add condition
+   query=$query" WHERE "
+   # add condition for step, i.e. step is not yet done
+   query=$query" ISNULL("$step"Status.fStartTime) "
+   query=$query" AND ISNULL("$step"Status.fStopTime) "
+   query=$query" AND ISNULL("$step"Status.fAvailable) "
+   query=$query" AND ISNULL("$step"Status.fReturnCode) "
+   # add requirement for production host in case it is needed
+   if [ "$1 " != " " ]
+   then 
+      query=$query" AND fProductionHostKEY=$2 "
+   fi
+   if ! echo $query | grep UPDATE >/dev/null 2>&1
+   then 
+      query=$query" GROUP BY "${prims[@]}
+   fi
+   # add condition for needs, i.e. that step is done
+   for (( k=0 ; k < ${#needs[@]} ; k++ ))
+   do
+      if [ $k -eq 0 ]
+      then 
+         query=$query" HAVING "
+      else
+         query=$query" AND "
+      fi
+      query=$query" COUNT(*)=COUNT(IF("
+      query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
+      query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
+      query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
+      query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
+      query=$query" , 1, NULL)) "
+   done
+}
+
+# function to get todolist
+#   returns the next or the list of next steps
+function gettodo()
+{
+   # reset the variable for the number of the next step 
+   process=
+   printprocesslog "DEBUG getting todo for step $step..."
+   getstepinfo
+   # get query
+   query=" SELECT "${prims[@]}
+   getstatusquery $2
+   # order by priority to the the number of the next step to be done
+   query=$query" ORDER BY "$step"Status.fPriority desc "
+   # add limitation in case only one or a limited number of 
+   #  processes should be executed
+   if [ "$1 " != " " ]
+   then 
+      query=$query" limit 0, $1 "
+   fi
+   # print query 
+   printprocesslog "DEBUG gettodo for step $step QUERY: "$query
+   # execute query 
+   process=`sendquery`
+   #if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
+   #then
+   #   printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
+   #   finish
+   #fi
+   # get numbers of next step from mysql result
+   if [ "$process" = "" ]
+   then
+      printprocesslog "DEBUG => nothing to do"
+      finish
+   else
+      primaries=( $process )
+      num=`expr ${#primaries[@]} / ${#prims[@]} `
+   fi
+}
+
+# function to get the number of processes which still have to be done
+function getstatus()
+{
+   # reset the variable for the number of steps to be done
+   numproc=0
+   getstepinfo
+   # get query
+   query=" SELECT "${prims[@]}
+   getstatusquery $1
+   # print query
+   printprocesslog "DEBUG getstatus for step $step QUERY: "$query
+   # execute query
+   #numproc=`sendquery `#cannot be done with sendquery, because of row counting
+   if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query " | wc -l`
+   then
+      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
+      echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
+      continue
+   fi
+}
+
+# function to set status of a process in the db
+function setstatus()
+{
+   # remark:
+   # this function does not include the 'Default' flag 
+   # for resetting steps
+
+   # for dowebplots (there are steps which have no entry in the DB)
+   if [ "$step" = "no" ]
+   then
+      return
+   fi
+   
+   # reset status values
+   starttime=NULL
+   stoptime=NULL
+   availtime=NULL
+   returncode=NULL
+   # evaluate the status values
+   case $@ in
+      start)   printprocesslog "DEBUG setstatus start"
+               starttime="Now()"
+               ;;
+       stop)   case $check in
+                  ok)  printprocesslog "DEBUB setstatus stop - ok"
+                       starttime=noreset
+                       stoptime="Now()"
+                       if [ "$processingsite" = "$storagesite" ]
+                       then
+                          availtime="Now()"
+                       fi
+                       ;;
+                  no)  printprocesslog "DEBUG setstatus stop - nothing new"
+                       check="ok"
+                       ;;
+                   *)  printprocesslog "DEBUG setstatus stop - failed"
+                       starttime=noreset
+                       stoptime="Now()"
+                       if [ "$processingsite" = "$storagesite" ]
+                       then
+                          availtime="Now()"
+                       fi
+                       if [ "$check" == "" ]
+                       then
+                          returncode=1
+                       else
+                          returncode=$check
+                       fi
+                       check="ok"
+                       ;;
+               esac
+               ;;
+          *)   printprocesslog "ERROR function setstatus got wrong variable"
+               finish
+               ;;
+   esac
+   
+   # get 
+   getstepinfo
+   
+   # get the influences from the steps.rc by evaluating the needs of all steps
+   othersteps=`grep $step $steps | grep -v '#' | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
+   
+   # get query
+   query=" UPDATE "
+   getalljoins
+   # set the status values according to the new status of the step
+   query=$query" SET "
+   if ! [ "$starttime" = "noreset" ]
+   then
+      query=$query" "$step"Status.fStartTime=$starttime, "
+   fi
+   query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
+   query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
+   # set also the status values of the influenced steps
+   for otherstep in $othersteps
+   do
+      query=$query", "$otherstep"Status.fStartTime=NULL "
+      query=$query", "$otherstep"Status.fStopTime=NULL "
+      query=$query", "$otherstep"Status.fAvailable=NULL "
+      query=$query", "$otherstep"Status.fReturnCode=NULL "
+      query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
+   done
+   # give the condition for which step the status values have to be set
+   query=$query" WHERE "
+   if [ "$s" = "" ]
+   then
+      s=0
+   fi
+   query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
+   for (( j=1 ; j < ${#prims[@]} ; j++ ))
+   do
+      query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
+   done 
+   # add additional query to allow for locking in db
+   if [ "$1" = "start" ]
+   then 
+      query=$query" AND ISNULL("$step"Status.fStartTime) "
+   fi
+   # add row count to know how many rows have been changed
+   query=$query"; SELECT ROW_COUNT();"
+   # print query
+   printprocesslog "DEBUG setstatus for step $step QUERY: "$query
+   # execute query
+   numchanged=`sendquery`
+   #if ! numchanged=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
+   #then
+   #   printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
+   #   finish
+   #fi
+   if [ $numchanged -gt 0 ]
+   then
+      printprocesslog "INFO successful set of status in DB."
+   else
+      # should not happen
+      printprocesslog "ERROR status in DB was already set by another process "
+   fi
+}
+
+
Index: trunk/DataCheck/crontab.coma
===================================================================
--- trunk/DataCheck/crontab.coma	(revision 13035)
+++ 	(revision )
@@ -1,13 +1,0 @@
-MAILTO=system@fact-project.org
-# cronjobs for coma (Wuerzburg)
-# change permissions for files and directories in /fact
-##########################################################################
-# permissions: 
-#   directories: u:+rwx  g:+rx  o:+rx -> 755
-#   files:       u:+rw   g:+r   o:+r  -> 644
-##########################################################################
-0 8,18 * * * /usr/bin/find /fact/raw/ -type d -exec /bin/chmod 755 {} \;
-0 8,18 * * * /usr/bin/find /fact/raw/ -type f -exec /bin/chmod 644 {} \;
-0 8,18 * * * /usr/bin/find /fact/aux/ -type d -exec /bin/chmod 755 {} \;
-0 8,18 * * * /usr/bin/find /fact/aux/ -type f -exec /bin/chmod 644 {} \;
-
Index: trunk/DataCheck/crontab.fact-transfer
===================================================================
--- trunk/DataCheck/crontab.fact-transfer	(revision 13035)
+++ 	(revision )
@@ -1,6 +1,0 @@
-MAILTO=daniela.dorner@unige.ch
-# cronjobs for user fact_opr (ISDC) on isdc-dl00 (machine for transfer)
-# do backup of auxiliary data to Wuerzburg
-0,30 * * * * if ! ps aux | grep BackupAuxToWue | grep -v grep >/dev/null 2>&1  ; then ~/DataCheck/BackupAuxToWue.sh; fi
-# do backup of raw data to Wuerzburg
-0 * * * * if ! ps aux | grep BackupRawToWue | grep -v grep >/dev/null 2>&1  ; then ~/DataCheck/BackupRawToWue.sh; fi
Index: trunk/DataCheck/crontab.fact01
===================================================================
--- trunk/DataCheck/crontab.fact01	(revision 13035)
+++ 	(revision )
@@ -1,8 +1,0 @@
-MAILTO=daniela.dorner@unige.ch
-# cronjobs for fact01 alias gate
-# check time offset of machine
-0 */6 * * * /home/fact/DataCheck/CheckNTPDate.sh
-# backup databases 
-0 8 * * * export AUTOMATIONSETUP="lp.fact" ; /home/fact/DB_Backup/Mars.svn.2011.08.05/datacenter/scripts/budb
-# cp databases backup to Wue
-10 8 * * * /home/fact/DataCheck/RsyncDBToWue.sh
Index: trunk/DataCheck/crontab.fact02
===================================================================
--- trunk/DataCheck/crontab.fact02	(revision 13035)
+++ 	(revision )
@@ -1,4 +1,0 @@
-MAILTO=daniela.dorner@unige.ch
-# cronjobs for fact02 alias aux
-# check time offset of machine
-0 */6 * * * /home/fact/SW.automatic.processing/DataCheck/CheckNTPDate.sh
Index: trunk/DataCheck/crontab.fact04
===================================================================
--- trunk/DataCheck/crontab.fact04	(revision 13035)
+++ 	(revision )
@@ -1,4 +1,0 @@
-MAILTO=daniela.dorner@unige.ch
-# cronjobs for fact04 alias gui
-# check time offset of machine
-0 */6 * * * /home/fact/SW.automatic.processing/DataCheck/CheckNTPDate.sh
Index: trunk/DataCheck/crontab.fact1
===================================================================
--- trunk/DataCheck/crontab.fact1	(revision 13035)
+++ 	(revision )
@@ -1,17 +1,0 @@
-MAILTO=daniela.dorner@unige.ch
-# cronjobs for fact1 alias data
-# check time offset of machine
-0 */6 * * * /home/fact/SW.automatic.processing/DataCheck/CheckNTPDate.sh
-# rsync data from fact2 to fact1
-*/30 * * * * if ! ps aux | grep RsyncRaw | grep -v grep >/dev/null 2>&1 ; then /home/fact/SW.automatic.processing/DataCheck/RsyncRaw.sh ; fi
-*/15 * * * * if ! ps aux | grep RsyncAux | grep -v grep >/dev/null 2>&1  ; then /home/fact/SW.automatic.processing/DataCheck/RsyncAux.sh ; fi
-# compress data on fact1
-42 * * * * if ! ps aux | grep ZipRawData | grep -v grep >/dev/null 2>&1  ; then /home/fact/SW.automatic.processing/DataCheck/ZipRawData.sh ; fi
-# check raw data on fact1
-42 * * * * if ! ps aux | grep CheckRawData | grep -v grep >/dev/null 2>&1  ; then /home/fact/SW.automatic.processing/DataCheck/CheckRawData.sh ; fi
-# fill auxiliary data on fact1
-21 8,10,12 * * * if ! ps aux | grep FillAuxData | grep -v grep >/dev/null 2>&1  ; then /home/fact/SW.automatic.processing/DataCheck/FillAuxData.sh ; fi
-# check disk usage for /loc_data
-0 * * * * /home/fact/SW.automatic.processing/DataCheck/CheckDU.sh
-# check processlog for WARN and ERROR
-10 0 * * * export AUTOMATIONSETUP="lp.fact"; /home/fact/SW.automatic.processing/Mars.CheObs.svn.2012.01.25/datacenter/scripts/checklogs
Index: trunk/DataCheck/crontab.fact2
===================================================================
--- trunk/DataCheck/crontab.fact2	(revision 13035)
+++ 	(revision )
@@ -1,8 +1,0 @@
-MAILTO=daniela.dorner@unige.ch
-# cronjobs for fact2 alias daq
-# check network status 
-*/15 * * * * /home/fact/SW.automatic.processing/DataCheck/CheckNetworkStatus.sh
-# check time offset of machine
-0 */6 * * * /home/fact/SW.automatic.processing/DataCheck/CheckNTPDate.sh
-# check disk usage for /loc_data
-0 * * * * /home/fact/SW.automatic.processing/DataCheck/CheckDU.sh