Index: trunk/MagicSoft/Mars/Changelog
===================================================================
--- trunk/MagicSoft/Mars/Changelog	(revision 9122)
+++ trunk/MagicSoft/Mars/Changelog	(revision 9123)
@@ -18,4 +18,82 @@
 
                                                  -*-*- END OF LINE -*-*-
+
+ 2008/08/15 Daniela Dorner
+
+   * datacenter/macros/plotdb.C, datacenter/macros/plotoptical.C,
+     datacenter/macros/plotstat.C, datacenter/macros/plotusage.C:
+     - added call plot*(TString path) needed for the script 
+       dowebplots
+     - added path to output file
+
+   * datacenter/macros/plotdb.C:
+     - added path to call plotdb(dataset)
+
+   * datacenter/macros/plotoptical.C:
+     - added path to call plotoptical(source)
+
+   * datacenter/macros/plotstat.C:
+     - fixed bug in query
+
+   * datacenter/macros/plotusage.C:
+     - adapted comment
+     - removed not needed calls
+
+   * datacenter/scripts/dowebplots:
+     - rewritten completely: script now handles not only creating 
+       web plots for ganymed, star and callisto, but also plotdb, 
+       plotusage, plotstat and plotoptical
+     - implemented steering via DB for ganymed, star and callisto to 
+       make sure that all plots are produced
+     - added rsync to this script
+     - via command line options it is possible to choose which plots 
+       should be produced
+     - redirect of the root error (used to output mean values) to 
+       txt files in the webdirectory
+     - added writing of csv file (read by tabs.php)
+
+   * datacenter/scripts/run.condor:
+     - added 'Arguments' (needed to submit dowebplots including
+       command line opticon to condor) 
+
+   * datacenter/scripts/scriptlauncher:
+     - removed makedir of $listpath and $lockpath (done in setup
+       already)
+     - exchanged algorithm to loop over scripts to allow for scripts
+       with command line option
+
+   * datacenter/scripts/sourcefile, datacenter/scripts/jobmanager:
+     - moved definition of $column from sourcefile to jobmanager
+       to allow for usage of the function getstatus by other scripts
+       than the jobmanager
+
+   * datacenter/scripts/sourcefile:
+     - removed definition of webpath (already defined in setup)
+     - removed making of $listpath (not needed anymore)
+     - added fail code for dowebplots (27)
+     - additional if-clause in the getstatus function to allow 
+       for using it even if no column is defined
+     - removed not needed 'numproc=0'
+
+   * resources/steps.rc:
+     - added steps for webplots (fWebCalib, fWebStar, fWebGanymed, 
+       fWebPlotDBGanymed)
+
+
+
+ 2008/08/14 Daniela Dorner
+
+   * datacenter/macros/plotusage.C:
+     - added (macro to plot condor usage from DB)
+
+
+
+ 2008/08/13 Daniela Dorner
+
+   * datacenter/scripts/setup:
+     - removed listpath (not used anymore)
+     - removed commented, not needed lines
+
+
 
  2008/08/13 Stefan Ruegamer
Index: trunk/MagicSoft/Mars/NEWS
===================================================================
--- trunk/MagicSoft/Mars/NEWS	(revision 9122)
+++ trunk/MagicSoft/Mars/NEWS	(revision 9123)
@@ -7,4 +7,18 @@
    * Added informations about the DC currents (fMinCurrents, fMedCurrents
      and fMaxCurrents)
+
+ ;automatic analyis
+
+   * Rewritten producing plots in the web. 
+     New plots, namely the output of plotoptical.C (extinction curve from 
+     KVA), plotstat.C (status of the automatic analysis (percentage of 
+     processed, analysed data etc.)) and plotusage.C (statistics of the 
+     condor usage in the data center) has been are available now next to the 
+     plots of plotdb.C
+     Mean values from the plots (output of the macro) are provided in a 
+     txt file now. 
+     The creation of the plots for callisto, star and ganymed in the web is 
+     now steered via database. Consequently the plots are faster and 
+     reliable up-to-date. 
 
 
Index: trunk/MagicSoft/Mars/datacenter/scripts/dowebplots
===================================================================
--- trunk/MagicSoft/Mars/datacenter/scripts/dowebplots	(revision 9122)
+++ trunk/MagicSoft/Mars/datacenter/scripts/dowebplots	(revision 9123)
@@ -18,7 +18,7 @@
 #
 #
-#   Author(s): Daniela Dorner  08/2005 <mailto:dorner@astro.uni-wuerzburg.de>
-#
-#   Copyright: MAGIC Software Development, 2000-2007
+#   Author(s): Daniela Dorner  08/2008 <mailto:dorner@astro.uni-wuerzburg.de>
+#
+#   Copyright: MAGIC Software Development, 2000-2008
 #
 #
@@ -31,4 +31,17 @@
 # files are produced. 
 #
+# The files from which plots are created can be devided into two categories:
+# general: plots needed only once per telescope
+# individual: plots done for each sequence / dataset
+#
+# For 'general' first the rootfiles are created with a macro from the 
+# database, then from this the ps via showplot, from this the pdf and the 
+# png files. 
+# For 'individual' running the macro is only needed for the plotdb for 
+# ganymed. The other steps are done accordingly. 
+#
+# Via command line options the plots can be selected. For more information
+# execute the script with the -h option. 
+#
 
 source `dirname $0`/sourcefile
@@ -38,103 +51,289 @@
 set -C
 
+
 scriptlog=$runlogpath/$program-$datetime.log
 date >> $scriptlog 2>&1
 
-while getopts p: opts
+echo $@  >> $scriptlog 2>&1
+
+# definition of the plots to be created
+generals=( plotdb optical condor status )
+individuals=( calib signal star ganymed gplotdb )
+# definition of the variables needed and varying for the plots
+names=(     plotdb     optical    condor   status     calib     signal     star     ganymed           gplotdb )
+pathnames=( plotdb     optical    condor   status  callisto   callisto     star     ganymed           ganymed )
+macros=(    plotdb plotoptical plotusage plotstat        no         no       no          no                no )
+filenames=( plotdb plotoptical plotusage plotstat     calib     signal     star     ganymed            plotdb )
+columns=(       no          no        no       no fWebCalib fWebSignal fWebStar fWebGanymed fWebPlotDBGanymed )
+digits=( 0 0 0 0 4 4 4 5 5 )
+
+# function to print the usage of the script
+function usage()
+{
+   echo "Usage: $0 [options]"
+   echo "options:"
+   echo -n "  --general    name  "
+   echo " where 'name' can be 'all' or one of the following: ${generals[@]} "
+   echo -n "  --individual name  "
+   echo " where 'name' can be 'all' or one of the following: ${individuals[@]} "
+   echo -n "  -h                 "
+   echo " print this usage "
+   echo "" 
+   echo "Remark: if no commandline option is given the plots are produced for everything." 
+   echo "" 
+   exit
+}
+
+# reading in the command line options
+general=( )
+individual=( )
+while [ "$1" ]
+do 
+   case $1 in 
+        --general) shift
+                   if [ "$1" = "all" ]
+                   then
+                      general=( ${generals[@]} )
+                   else 
+                      general=( $general $1 )
+                   fi
+                   ;;
+     --individual) shift
+                   if [ "$1" = "all" ]
+                   then
+                      individual=( ${individuals[@]} )
+                   else 
+                      individual=( $individual $1 )
+                   fi
+                   ;;
+              -h)  usage 
+                   exit
+                   ;;
+               *)  echo "unknown option $1 " >> $scriptlog 2>&1
+                   usage  >> $scriptlog 2>&1
+                   exit
+                   ;;
+   esac
+   shift
+done
+
+if [ ${#general[@]} -eq 0 ] || [ ${#individual[@]} -eq 0 ]
+then
+   all=( ${names[@]} )
+else
+   all=( ${general[@]} ${individual[@]} ) 
+fi
+
+echo "For the following plots will be produced: "${all[@]} >> $scriptlog 2>&1
+
+
+
+# rsync subsystemdata, sequence and dataset files
+echo "do rsync for subsystem files" >> $scriptlog 2>&1
+#rsync -avn --delete $subsystempath/ $webpath/subsystemdata >> $scriptlog 2>&1
+
+echo "do rsync for sequence files" >> $scriptlog 2>&1
+#rsync -avn --delete $sequencepath/ $webpath/sequences >> $scriptlog 2>&1
+
+echo "do rsync for dataset files" >> $scriptlog 2>&1
+#rsync -avn --delete $datasetpath/ $webpath/datasets >> $scriptlog 2>&1
+
+
+# using hour for lock => after 24 no new script is 
+#  started in case the previous is still running
+hour=`date +%H`
+lockfile=$lockpath/${program}-${hour}h.txt
+checklock  >> $scriptlog 2>&1
+
+count=0
+# loop over all scripts in $names
+while [ 1 -gt 0 ]
 do
-   case $opts in
-      p)  type=$OPTARG
-          echo "got programname: $type"  >> $scriptlog 2>&1
-          ;;
-      ?)  echo "usage: $(basename $0) -p programname"  >> $scriptlog 2>&1
-          ;;
-   esac
-done
-
-if [ "$type" = "" ]
-then 
-   echo "no program name given -> exit"  >> $scriptlog 2>&1
-   echo "usage: $(basename $0) -p programname"   >> $scriptlog 2>&1
-   date  >> $scriptlog 2>&1
-   printprocesslog "INFO finished $0"
-   exit
-fi
-
-# check if script is already running
-lockfile=$lockpath/lock-$program-$type.txt
-checklock  >> $scriptlog 2>&1
-
-#finding all rootfiles in the data directory, that were modified in the last 3 days
-#this are all statusdisplays 
-rootfiles=`find $datapath/$type/ -maxdepth 10 -name '*.root' -mtime -3 | grep -v '_I_' | grep -v '_Y_' | grep -v 'summary'`
-
-#exit if no rootfiles are found
-if [ "$rootfiles" = "" ]
-then 
-   echo "nothing to do -> exit"  >> $scriptlog 2>&1
-   finish >> $scriptlog 2>&1
-fi
-
-
-cd $mars
-
-#produce plots for each rootfile
-printprocesslog "INFO procduing plots for each $type rootfile"
-for rootfile in ${rootfiles[@]}
-do 
-   date >> $scriptlog 2>&1
-   #get names of the psfile and the 
-   #pngfiles (1 per tab in the statusdiplay)
-   psfile=`echo $rootfile | sed -e 's/.root$/.ps/g'`
-   tabfile=`echo $rootfile | sed -e 's/.root$/-tab/g'`
-   echo "rootfile: $rootfile" >> $scriptlog 2>&1
-   echo "psfile:   $psfile"   >> $scriptlog 2>&1
-   echo "tabfile:  $tabfile"  >> $scriptlog 2>&1
+   check="ok"
+   # get needed variables from the arrays
+   name=${names[$count]}
+   macro=${macros[$count]}
+   pathname=${pathnames[$count]}
+   filename=${filenames[$count]}
+   column=${columns[$count]}
+   digit=${digits[$count]}
    
-   #get date of root- and psfile
-   daterootfile=`date +%Y%m%d -r $rootfile` >> $scriptlog 2>&1
-   datepsfile=`date +%Y%m%d -r $psfile` >> $scriptlog 2>&1
-   if [ "$datepsfile" = "" ]
+   count=`expr $count + 1`
+   if [ $count -gt ${#names[@]} ]
    then
-      echo "date of psfile is empty, i.e. the file $psfile doesn't exist" >> $scriptlog 2>&1
-      echo " -> setting date to 0 and producing psfile..." >> $scriptlog 2>&1
-      datepsfile=0
-   fi
-
-   echo "checking date..."  >> $scriptlog 2>&1
-   echo "date of rootfile: $daterootfile" >> $scriptlog 2>&1
-   echo "date of psfile:   $datepsfile"   >> $scriptlog 2>&1
-   #if the psfile is newer than the rootfile
-   #no plots have to be done -> continue
-   if [ "$datepsfile" -gt "$daterootfile" ] >> $scriptlog 2>&1
-   then
-      echo "psfile is newer than rootfile -> continue " >> $scriptlog 2>&1
+      count=0
       continue
    fi
-
-   printprocesslog "INFO procduing plots for $rootfile"
-   echo "producing psfile..." >> $scriptlog 2>&1
-   ./showplot -b --save-as-ps=$psfile $rootfile >> $scriptlog 2>&1
-
-   echo "creating temporary dir for pstoimg..." >> $scriptlog 2>&1
-   temppath=`dirname $rootfile | sed -e 's/\/magic\/data//'`
-   tempwebplotspath=/tmp/pstoimgtmp$temppath
-   makedir $tempwebplotspath >> $scriptlog 2>&1
-
-   echo "converting plots to png..." >> $scriptlog 2>&1
-   pstoimg -antialias -aaliastext -interlaced -flip r270 -density 100 -tmp $tempwebplotspath -type png -multipage -out=$tabfile $psfile >> $scriptlog 2>&1
+   # find out if these plots have to be created
+   #  by looping over the complete array $all and comparing the entries to those of the array $names
+   for (( a=0 ; a <= ${#all[@]} ; a++ ))
+   do 
+      # exit the loop if the entries agree
+      #  i.e. produce plots for this entry
+      if [ "${all[$a]}" = "$name" ]
+      then
+         break
+      fi
+      # continue if the end of array is not yet reached
+      if [ $a -lt ${#all[@]} ]
+      then
+         continue
+      fi
+      # exit both loops if no entry in the array is left
+      #  i.e. there is nothing to do anymore
+      if [ ${#all[@]} -eq 0 ]
+      then
+         break 2
+      fi
+      # continue with the next entry in the array $names
+      #  in case the end of the array $all is reached without agreement
+      continue 2
+   done
+
+   # do things specific to 'general' and 'individual'
+   if [ "$macro" = "no" ]
+   then 
+      # 'individual'
+      # remove name from list in case there is nothing left to do
+      getstatus >> $scriptlog 2>&1
+      if [ "$numproc" = "" ]
+      then 
+         unset all[$a]
+         all=( ${all[@]} )
+         continue
+      fi
+      # get one sequence or dataset from the database for which plots have to be produced
+      gettodo "1" >> $scriptlog 2>&1
+      num=${primaries[0]}
+      echo "executing $name for number "$num  >> $scriptlog 2>&1
+      # lock the process
+      lockfile=$lockpath/$program-$name-$num.txt
+      checklock continue  >> $scriptlog 2>&1
+      setstatus "start" >> $scriptlog 2>&1
+      # set needed file and path names
+      pathpart=$pathname/`printf %08d $num | cut -c 0-${digit}`/`printf %08d $num`
+      inpath=$datapath/$pathpart
+      outpath=$webpath/$pathpart
+      tmppath=/tmp/pstoimg$pathpart
+      rootfile=$inpath/$filename`printf %08d $num`.root
+      psfile=$inpath/$filename`printf %08d $num`.ps
+      csvfile=$inpath/$filename`printf %08d $num`.csv
+      pdffile=$outpath/$filename`printf %08d $num`.pdf
+      # rsync
+      makedir $outpath >> $scriptlog 2>&1
+      rsync -aLv --exclude=20[01][0-9]*_[YI]_*[.]root $inpath/ $outpath >> $scriptlog 2>&1
+
+      # for ganymed not only the ganymed*.root is processed but also a plotdb.root for the 
+      #  dataset. therefore the macro plotdb.C has to be run in this case
+      if [ "$name" = "gplotdb" ]
+      then
+         # overwrite needed file and path names
+         rootfile=$inpath/$filename.root
+         psfile=$inpath/$filename.csv
+         pdffile=$outpath/$filename.pdf
+         datasetfile=$datasetpath/`printf %08d $num | cut -c 0-${digit}`/dataset`printf %08d $num`.txt
+         
+         echo "check1=root -q -b $macrospath/plotdb.C+\(\"$datasetfile\"\,\"$inpath/\"\) | tee -a $scriptlog | intgrep" >> $scriptlog 2>&1
+         check1=`root -q -b $macrospath/plotdb.C+\("\"$datasetfile\""\,"\"$inpath/\""\) | tee -a $scriptlog | intgrep`
+         case $check1 in 
+            1)   echo " check1=$check1 -> everything ok" >> $scriptlog 2>&1
+                 printprocesslog "INFO rootfile $rootfile successfully produced"
+                 ;;
+            *)   echo " check1=$check1 -> ERROR -> couldn't create plots -> exit" >> $scriptlog 2>&1
+                 printprocesslog "ERROR producing rootfile $rootfile failed"
+                 com=$Fdowebplots
+                 check=$check1
+                 ;;
+         esac
+      fi
+   else
+      # 'general'
+      echo "executing $name "  >> $scriptlog 2>&1
+      # remove 'general' from list as they need to be executed only once
+      unset all[$a]
+      all=( ${all[@]} )
+      # lock process
+      lockfile=$lockpath/$program-$name.txt
+      checklock continue >> $scriptlog 2>&1
+      # set needed file and path names
+      outpath=$webpath/$pathname
+      tmppath=/tmp/$pathname
+      rootfile=$outpath/$filename.root
+      psfile=$outpath/$filename.csv
+      pdffile=$outpath/$filename.pdf
+      txtfile=$outpath/$filename.txt
+      
+      echo "check2=root -q -b $macrospath/$macro.C+\(\"$outpath/\"\) 2>> $txtfile | tee -a $scriptlog | intgrep" >> $scriptlog 2>&1
+      check2=`root -q -b $macrospath/$macro.C+\("\"$outpath/\""\) 2>> $txtfile | tee -a $scriptlog | intgrep`
+      case $check2 in 
+         1)   echo " check2=$check2 -> everything ok" >> $scriptlog 2>&1
+              printprocesslog "INFO rootfile $rootfile successfully produced"
+              ;;
+         *)   echo " check2=$check2 -> ERROR -> couldn't produce root file $rootfile  -> exit" >> $scriptlog 2>&1
+              printprocesslog "ERROR producing rootfile $rootfile failed"
+              com=$Fdowebplots
+              check=$check2
+              ;;
+      esac
+   fi
    
-   echo "removing temporary dir..." >> $scriptlog 2>&1
-   rmdir -pv $tempwebplotspath >> $scriptlog 2>&1
+   # in case no error occured, the plots are produced from the rootfile
+   if [ "$check" = "ok" ]
+   then
+      printprocesslog "INFO procduing plots (psfile and png) for $rootfile"
+      if ! ls $rootfile >/dev/null 2>&1
+      then
+         echo "rootfile $rootfile does not exist" >> $scriptlog 2>&1
+         printprocesslog "ERROR rootfile $rootfile does not exist"
+         com=$Fdowebplots
+         check=3
+      else
+         echo "producing psfile and csvfile..." >> $scriptlog 2>&1
+         if ! ./showplot -b --save-as-ps=$psfile --save-as-csv=$csvfile $rootfile >> $scriptlog 2>&1
+         then
+            com=$Fdowebplots
+            check=4
+         else
+            echo "creating temporary directory for pstoimg..." >> $scriptlog 2>&1
+            makedir $tmppath >> $scriptlog 2>&1
+            
+            echo "converting plots to png..." >> $scriptlog 2>&1
+            if ! pstoimg -antialias -aaliastext -interlaced -flip r270 -density 100 -tmp $tmppath -out=$outpath/$name -type png -multipage $psfile >> $scriptlog 2>&1
+            then 
+               printprocesslog "WARN could not create png files for $psfile "
+               com=$Fdowebplots
+               check=5
+            fi
+            
+            echo "removing temporary directory..." >> $scriptlog 2>&1
+            rmdir -pv $tmppath >> $scriptlog 2>&1
+            
+            echo "converting $psfile to pdf..." >> $scriptlog 2>&1
+         fi
+         
+         if ps2pdf $psfile $pdffile >> $scriptlog 2>&1
+         then 
+            rm -v $psfile  >> $scriptlog 2>&1
+         else
+            printprocesslog "WARN could not convert $psfile to $pdffile"
+            com=$Fdowebplots
+            check=6
+         fi
+      fi
+   fi
    
-   echo "converting $psfile to pdf..." >> $scriptlog 2>&1
-   if ps2pdf $psfile >> $scriptlog 2>&1
-   then 
-      rm -v $psfile  >> $scriptlog 2>&1
-   else
-      printprocesslog "WARN could not convert $psfile to pdf"
-   fi
+   setstatus "stop" >> $scriptlog 2>&1
+   
+   rm -v $lockfile >> $scriptlog 2>&1
+
+   sleep 2
 done
 
+# rsync for complete directory delete removed sequences
+echo "do rsync for callisto, star and ganymed files" >> $scriptlog 2>&1
+rsync -aLvn --delete --exclude=*/*/20[01][0-9]*_[YI]_*[.]root --exclude=*/*.ps $datapath/{callisto,star,ganymed}/ $webpath/callisto >> $scriptlog 2>&1
+
+lockfile=$lockpath/${program}-${hour}h.txt
 finish >> $scriptlog 2>&1
 
+