Index: trunk/DataCheck/QuickLook/Step2b.sh
===================================================================
--- trunk/DataCheck/QuickLook/Step2b.sh	(revision 16987)
+++ trunk/DataCheck/QuickLook/Step2b.sh	(revision 16988)
@@ -4,114 +4,140 @@
 printprocesslog "INFO starting $0"
 
-#root=/opt/root_svn/bin/thisroot.sh
-#source $root
-#factpath=/home/fact/FACT++.in-run-fad-loss
+# fixme: get sources from DB
+sources=( 1 2 5 7 )
+#sources=( 1 2 5 )
+numganymeds=0
 
-#anapath=/loc_data/analysis
-
-sources=( 1 2 5 7 )
-numganymeds=0
+printprocesslog "INFO get all nights."
+query="SELECT fNight FROM RunInfo WHERE not isnull(fSourceKey) and fNight > 20130301 GROUP BY fNight ORDER BY fNight DESC"
+nights=( `sendquery` )
 
 for source in ${sources[@]}
 do 
-   # getting all nights
-   printprocesslog "INFO get all nights for source "$source
-   dirs=( `find $anapath/$source/datasets_run -maxdepth 3 -mindepth 3 -type d 2>/dev/null | sort` )
-   #nights=1
-   # write data set file
-   dspath=$anapath/$source/datasets_night
+   dspath=$anapath/datasets_night/$source
    makedir $dspath
-   for (( i=1 ; i<=${#dirs[@]} ; i++ ))
+done
+
+for night in ${nights[@]}
+do 
+   echo $night
+   for source in ${sources[@]}
    do 
-      numdsfiles=`find ${dirs[$i-1]} -type f | wc -l`
-      imgpath=`echo ${dirs[$i-1]} | sed -e 's/datasets_run/star/'`
-      numimgfiles=`find $imgpath -type f -name '*_I.root' | wc -l`
-      echo ${dirs[$i-1]}" "$numdsfiles" "$numimgfiles
-      night=`echo ${dirs[$i-1]} | cut -d/ -f6-8 | sed -e 's/\///g'`
-      echo $night
+      dspath=$anapath/datasets_night/$source
       dsfile=$dspath"/dataset"$night".txt"
-      numold=`cat $dsfile | wc -l`
-      echo ${dirs[$i-1]}" "$numdsfiles" "$numimgfiles" "$numold
-      echo $dsfile
-      printprocesslog "INFO writing dataset file "$dsfile
-      echo "INFO writing dataset file "$dsfile
-      #cat ${dirs[$i-1]}/dataset*.txt
-      cat ${dirs[$i-1]}/dataset*.txt > $dsfile
-      # process only if ganymedlogfiles is not yet there
-      logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
-      echo $logfile
-      # this does not work as there might be dataset files already pr
-      #if [ $numdsfiles -ne $numimgfiles ]
-      if [ $numold -lt $numimgfiles ]
+      if ! ls $dsfile >/dev/null 2>&1
       then
-         printprocesslog "INFO remove $logfiles (new files added to dataset) " 
-         rm $logfile
+         echo "# source "$source" for night "$night > $dsfile
+         linesold=0
+      else
+         lines=( `wc -l $dsfile` )
+         linesold[$source]=${lines[0]}
       fi
-      if ! ls $logfile >/dev/null 2>&1
+   done
+
+   inpath=$anapath/star/`echo $night | cut -c 1-4`/`echo $night | cut -c 5-6`/`echo $night | cut -c 7-8`
+   #files=`find $inpath -name '*_I.root'`
+   files=( `ls $inpath/*_I.root 2>/dev/null` )
+   if [ ${#files[@]} -eq 0 ]
+   then
+      continue
+   fi
+   echo "files: "${#files[@]}
+   for file in ${files[@]}
+   do 
+      run=`basename $file | cut -d_ -f2`
+      query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunId="$run" GROUP BY fSourceKey"
+      source=`sendquery`
+      if [ $source -eq 0 ]
       then
-         echo "`dirname $0`/RunGanymed.sh $source $dsfile"
-         `dirname $0`/RunGanymed.sh $source $dsfile
-         numganymeds=`echo " $numganymeds + 1 " | bc -l`
+         continue
       fi
-   done 
+      dspath=$anapath/datasets_night/$source
+      dsfile=$dspath"/dataset"$night".txt"
+      if ! grep `basename $file` $dsfile >/dev/null 2>&1
+      then 
+         if echo $resulttable1 | grep ISDC >/dev/null 2>&1
+         then 
+            query="SELECT LPAD(fSequenceID, 3, 0) FROM RunInfo WHERE fNight="$night" AND fRunID="$run
+            seq=`sendquery`
+            merpplog=`dirname $file`"/"$night"_"$seq"-merpp.log"
+         else 
+            merpplog=`echo $file | sed -e 's/_I.root/-merpp.log/'`
+         fi
+         echo $merpplog	
+         stillrunning=`find $merpplog -mmin -1 2>/dev/null`
+         if [ -e $merpplog ] && [ "$stillrunning" == "" ]
+         then
+            echo `dirname $file`" "`basename $file` >> $dsfile
+            echo `dirname $file`" "`basename $file`
+         else
+            printprocesslog "INFO merpp still running - wait with file "$file
+            continue
+         fi
+      fi
+   done
    
-#   # get list of dataset files for 1 night as starting point
-#   printprocesslog "INFO get list of dataset files for "$nights" nights"
-#   files=( `find $dspath -type f -name 'dataset*.txt' | sort` )
-#   nightsold=1
-#   # loop over data set files
-#   # copy always 2 in one new file for double observation time
-#   while [ ${#files[@]} -gt 1 ]
-#   do
-#      # get double observation time and path for datasets
-#      nights=`echo " $nightsold + $nightsold " | bc -l `
-#      dspathold=$dspath
-#      dspath=$anapath"/"$source"/datasets"`printf %04d $nights`"nights"
-#      makedir $dspath
-#      # loop over already existing data set files
-#      for (( i=0 ; i<=${#files[@]} ; i++ ))
-#      do 
-#         # number of 1st and 2nd old dataset file
-#         num1=`echo " $i + $i + 1" | bc -l `
-#         num2=`echo " $i + $i + 2 " | bc -l `
-#         # number of new dataset file
-#         dsnum=`echo " $i + 1 " | bc -l `
-#         # continue only if 2 dataset files are left
-#         if ! [ ${files[$num1]} ]
-#         then
-#            break
-#         fi
-#         ds1=$dspathold"/dataset"$num1".txt"
-#         ds2=$dspathold"/dataset"$num2".txt"
-#         dsfile=$dspath"/dataset"$dsnum".txt"
-#         # copy two old into one new dataset file
-#         printprocesslog "INFO writing dataset "$dsfile
-#         echo "INFO writing dataset "$dsfile
-#         cat $ds1 > $dsfile
-#         cat $ds2 >> $dsfile
-#         # process only if ganymedlogfiles is not yet there
-#         logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
-#         if ! ls $logfile  >/dev/null 2>&1
-#         then
-#            `dirname $0`/RunGanymed.sh $source $dsfile
-#            numganymeds=`echo " $numganymeds + 1 " | bc -l`
-#         fi
-#      done
-#      # get list of new dataset files
-#      printprocesslog "INFO get list of dataset files for "$nights" nights"
-#      files=( `find $dspath -type f -name 'dataset*.txt' | sort` )
-#      nightsold=$nights
-#   done
-   
-    echo "found "$numganymeds" ganymeds."
-   #if [ $numganymeds -gt 0 ]
-   #then 
-   #    if ! ps aux | grep Step3 | grep -v grep >/dev/null 2>&1
-   #    then 
-   #      `dirname $0`/Step3.sh nights &
-   #    fi
-   #   numganymeds=0
-   #fi
-   
+   for source in ${sources[@]}
+   do 
+      dspath=$anapath/datasets_night/$source
+      dsfile=$dspath"/dataset"$night".txt"
+      if ls $dsfile >/dev/null 2>&1
+      then
+         lines=( `wc -l $dsfile` )
+         linesnew[$source]=${lines[0]}
+         if [ ${lines[0]} -le 1 ]
+         then
+            # not ideal - what if number of files stays the same, but files change?
+            rm -v $dsfile
+            continue
+         fi
+         echo "new: "${linesnew[$source]}
+         echo "old: "${linesold[$source]}
+         if [ ${linesnew[$source]} -le ${linesold[$source]} ]
+         then
+            echo "nothing new"
+            continue
+         else
+            ls $dsfile
+         fi
+      fi
+      query="SELECT Round(fRightAscension,6), Round(fDeclination,6) from source WHERE fSourceKey="$source
+      coordinates=( `sendquery` )
+      if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
+      then
+         printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
+         #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
+         continue
+      fi
+      if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
+      then
+         printprocesslog "WARN coordinates "${coordinates[@]}
+         #echo "WARN coordinates "${coordinates[@]}
+         continue
+      fi
+      if ls $dsfile >/dev/null 2>&1
+      then
+         outpath=$anapath/ganymeds_night/$source
+         logfile=$outpath"/ganymed"$night".log"
+         echo $logfile
+         #ls $logfile
+         if ! ls $logfile >/dev/null 2>&1
+         then
+            ls $dsfile
+            if [ "$qlasge" = "yes" ]
+            then
+               printprocesslog "$sgepath/qsub -b y -q fact_short -v AUTOMATIONSETUP=$AUTOMATIONSETUP -v dsfile=$dsfile -v ra=${coordinates[0]} -v dec=${coordinates[1]} -e bla.txt -o bla.txt `dirname $0`/RunGanymed.sh"
+               $sgepath/qsub -b y -q fact_short -v AUTOMATIONSETUP=$AUTOMATIONSETUP -v dsfile=$dsfile -v ra=${coordinates[0]} -v dec=${coordinates[1]} -e bla.txt -o bla.txt `dirname $0`/RunGanymed.sh
+            else
+               `dirname $0`/RunGanymed.sh $dsfile ${coordinates[0]} ${coordinates[1]}
+            fi
+            #echo "`dirname $0`/RunGanymed.sh $source $dsfile"
+            #`dirname $0`/RunGanymed.sh $source $dsfile
+            numganymeds=`echo " $numganymeds + 1 " | bc -l`
+         fi
+      fi
+   done
+
+   echo "found "$numganymeds" ganymeds."
 done