source: trunk/MagicSoft/Mars/datacenter/scripts/runganymed@ 7222

#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2004
#
#
# ========================================================================
#
#

user=`whoami`
source /home/$user/Mars/datacenter/scripts/sourcefile

set -C

table=DataSetProcessStatus
column=fGanymed
date=NULL
datetime=`date +%F-%H-%M-%S`
year=`date +%Y`
pno=24 # number of processes, i.e. number of todo-files

todofile=$listpath/ToDo-$table-$column
getstatuslogpath=$logpath/getstatus/ganymed/$year
getstatuslog=$getstatuslogpath/getstatus-ganymed-$datetime.log

scriptlogpath=$logpath/run/ganymed/`date +%Y/%m/%d`
if [ ! -d $scriptlogpath ]
then
  mkdir -pv $scriptlogpath 
  if [ ! -d $scriptlogpath ]
  then 
     echo "could not make scriptlogpath "$scriptlogpath 
     exit
  fi
fi

scriptlog=$scriptlogpath/runganymed-$datetime.log

date >> $scriptlog 2>&1


if [ ! -d $getstatuslogpath ]
then
  mkdir -pv $getstatuslogpath >> $scriptlog 2>&1
  if [ ! -d $getstatuslogpath ]
  then 
     echo "could not make getstatuslogpath "$getstatuslogpath >> $scriptlog 2>&1
     date  >> $scriptlog 2>&1
     exit
  fi
fi

cd $mars

date > $lockpath/lock-getting-ganymed-list.txt >> $scriptlog 2>&1
checklock0=$?
case $checklock0 in 
    0)   echo "checklock0=$checklock0 -> continue " >> $scriptlog 2>&1;;
    1)   echo "checklock0=$checklock0 -> file exists " >> $scriptlog 2>&1
         echo "-> getting list for ganymed is running -> exit" >> $scriptlog 2>&1
         date  >> $scriptlog 2>&1
         exit;;
    *)   echo "checklock0=$checklock0 -> something went completely wrong" >> $scriptlog 2>&1;;
esac

echo "checking if other todo-files are there" >> $scriptlog 2>&1
if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
then 
   echo "other file(s) on disk " >> $scriptlog 2>&1
   echo " -> choose one file and run ganymed" >> $scriptlog 2>&1
else
   echo "getting list..." >> $scriptlog 2>&1
   check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
   
   case $check0 in 
       1)   echo "check0=$check0 -> everthing ok -> run ganymed" >> $scriptlog 2>&1;;
       *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
   esac
   echo "cutting to-do-file" >> $scriptlog 2>&1
   echo "getting no of lines" >> $scriptlog 2>&1
   lines=`cat $todofile.txt | wc -l`
   echo "number of lines: "$lines >> $scriptlog 2>&1
   while (( "$lines" < "$pno" ))
   do
      echo "# of lines ($lines) < # of processes ($pno) "
      pno=`expr $pno / 2` 
   done
   echo "pno: "$pno >> $scriptlog 2>&1
   nofiles=`expr $lines / \( $pno - 1 \)`
   echo "number of files: "$nofiles >> $scriptlog 2>&1
   echo "deviding todo-file" >> $scriptlog 2>&1
   for (( j=1 ; j <= $pno ; j++ ))
   do 
     begin=$(echo "1 + ( ($j - 1) * $nofiles)" | bc -l)
     end=$(echo "$begin + $nofiles - 1" | bc -l)
     echo "begin: "$begin >> $scriptlog 2>&1
     echo "end: "$end >> $scriptlog 2>&1
     file=${todofile}-${j}.txt
     echo "file: "$file >> $scriptlog 2>&1
     sed -ne ''"$begin"','"$end"'w '"$file"'' $todofile.txt
   done

   rm -v $todofile.txt >> $scriptlog 2>&1
fi

rm -v $lockpath/lock-getting-ganymed-list.txt >> $scriptlog 2>&1

nr=bla
echo "finding the right todo-file" >> $scriptlog 2>&1
for (( i = 1; i <= $pno ; i++ ))
do 
   if ! ls $todofile-$i.txt >> $scriptlog 2>&1
   then 
      echo "file is not on disk -> continue" >> $scriptlog 2>&1
      continue
   fi
   date > $lockpath/lock-$table-$column-$i.txt >> $scriptlog 2>&1 
   checklock=$?
   case $checklock in 
       0)   echo "checklock=$checklock -> setting number" >> $scriptlog 2>&1
            nr=${i}
            break;;
       1)   echo "checklock=$checklock -> file exists -> continue" >> $scriptlog 2>&1;;
       *)   echo "checklock=$checklock -> something went completely wrong" >> $scriptlog 2>&1;;
   esac
done

case $nr in
   bla)      echo "everything is beeing processed -> exit" >> $scriptlog 2>&1
             date  >> $scriptlog 2>&1
             exit;;
   12345678) echo "process nr: "$nr >> $scriptlog 2>&1;;
esac


datasets=(`cat $todofile-$nr.txt`)

if [ "$datasets" = "" ]
then 
   echo "nothing to do -> exit"  >> $scriptlog 2>&1
   rm -v $todofile-$nr.txt >> $scriptlog 2>&1
   rm -v $lockpath/lock-$table-$column-$nr.txt >> $scriptlog 2>&1
   date  >> $scriptlog 2>&1
   exit
fi

echo "datasets: "${datasets[@]} >> $scriptlog 2>&1

for dataset in ${datasets[@]}
do 
  no=`printf %08d $dataset | cut -c 0-5`
  no2=`printf %08d $dataset`
  outpath="$datapath/ganymed/$no/$no2"
  echo "outpath: "$outpath >> $scriptlog 2>&1
  if [ ! -d $outpath ]
  then
    mkdir -pv $outpath >> $scriptlog 2>&1
    if [ ! -d $outpath ]
    then 
       echo "could not make outpath $outpath -> continue " >> $scriptlog 2>&1
       continue
    fi
  fi
  datasetfile="/magic/datasets/$no/dataset$no2.txt"
  echo "datasetfile: "$datasetfile >> $scriptlog 2>&1
  wobble=`grep 'WobbleMode:' $datasetfile` >> $scriptlog 2>&1 
  wobble2=`echo $wobble | grep ^\#` >> $scriptlog 2>&1 
  if [ "$wobble2" = "" ]
  then 
     mode="wobble" >> $scriptlog 2>&1 
  else
     mode="onoff" >> $scriptlog 2>&1 
  fi
  ganymedrc=/magic/datacenter/setup/ganymed/ganymed_$mode.rc
  
  echo "run ganymed..." >> $scriptlog 2>&1
  ./ganymed -b -q -v4 -f --sum --config=$ganymedrc --log=$outpath/ganymed$no2.log --html=$outpath/ganymed$no2.html --out=$outpath $datasetfile  2>> $scriptlog> /dev/null
  check1=$?

  case $check1 in
     0)   echo "check1=$check1 -> everthing ok " >> $scriptlog 2>&1
          echo "-> inserting the status for ganymed for dataset $dataset into the db" >> $scriptlog 2>&1
          setstatuslogpath=$logpath/setstatus/ganymed/$no
          if [ ! -d $setstatuslogpath ]
          then
             mkdir -pv $setstatuslogpath >> $scriptlog 2>&1
             if [ ! -d $setstatuslogpath ]
             then 
                echo "could not make setstatuslogpath "$setstatuslogpath >> $scriptlog 2>&1
                continue
             fi
          fi
          setstatuslog=$setstatuslogpath/setstatus-ganymed-$no2.log
          check4=`root -q -b $macrospath/setstatus.C+\("\"$dataset\""\,"\"$table\""\,"\"$column\""\,"\"Now()\""\) | tee $setstatuslog | grep int | sed -e 's/(int)//'`
          case $check4 in
             1)   echo "check4=$check4 -> everthing ok, status has been set" >> $scriptlog 2>&1;;
             *)   echo "check4=$check4 -> ERROR -> step could not be set" >> $scriptlog 2>&1;;
          esac
          ;;
     *)   echo "check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
  esac
done

rm -v $todofile-$nr.txt >> $scriptlog 2>&1
rm -v $lockpath/lock-$table-$column-$nr.txt >> $scriptlog 2>&1

set +C

date  >> $scriptlog 2>&1