Index: trunk/DataCheck/QuickLook/Step1.sh
===================================================================
--- trunk/DataCheck/QuickLook/Step1.sh	(revision 15380)
+++ trunk/DataCheck/QuickLook/Step1.sh	(revision 15381)
@@ -4,20 +4,9 @@
 printprocesslog "INFO starting $0"
 
-#root=/opt/root_svn/bin/thisroot.sh
-#source $root
-#factpath=/home/fact/FACT++.in-run-fad-loss
-
-#anapath=/loc_data/analysis
-
-#rerun="yes"
-#rerun="no"
-# not needed anymore as the variable is set automatically 
-#   depending on whether the rawfile existed locally, 
-#   i.e. depending on whether the rsync in needed
-# still needed in case all rawfiles have been rsynced already
-
 # get date (before 18h there is no new data to be processed)
 datepath=`date --date="-18HOUR" +%Y/%m/%d`
-#datepath="2013/03/24"
+date=`date --date="-18HOUR" +%Y%m%d`
+#datepath="2013/04/17"
+#date="20130417"
 printprocesslog "INFO processing "$datepath
 night=`echo $datepath | sed -e 's/\///g'`
@@ -71,6 +60,4 @@
 
 printprocesslog "INFO get lists of raw files on newdaq and daq"
-#files=( `ssh newdaq "find $rawpath -type f 2>/dev/null | sort"` )
-#files=( `ssh 172.16.100.100 "find $rawpath -type f 2>/dev/null | sort"` )
 files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
 if [ ${#files[@]} -eq 0 ]
@@ -80,22 +67,15 @@
 fi
 fileslocal=( `find $rawpath -type f | sort` )
+callistofiles=( `find $anapath -type f -name $date*-calibration.log | sort` )
+numdataruns=0
 
 # create raw directory on daq, if not yet there
 makedir $rawpath
 
-
-while [ ${#fileslocal[@]} -lt ${#files[@]} ] || [ "$rerun" = "yes" ]
+echo ${#fileslocal[@]}" "${#files[@]}" "${#callistofiles[@]} 
+
+while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ]
 do
-   ##if ssh newdaq "ls $drivefile"
-   #if ls $drivefilenewdaq
-   #then 
-   #   printprocesslog "INFO rsync "$drivefilenewdaq
-   #   #if ! rsync -av --progress newdaq:$drivefile $drivefile
-   #   #if ! rsync -av --progress 172.16.100.100:$drivefile $drivefile
-   #   if ! rsync -av --stats $drivefilenewdaq $drivefile
-   #   then 
-   #      printprocesslog "WARN rsync of "$drivefilenewdaq" failed."
-   #   fi
-   #fi
+   numdataruns=0
    rsync_aux_file $drivefilenewdaq $drivefile
    
@@ -110,10 +90,6 @@
       then
          nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
-         #tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
-         #tstop=`ssh 172.16.100.100 "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
          tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
       else
-         #tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
-         #tstop=`ssh 172.16.100.100 "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
          tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
       fi
@@ -121,8 +97,5 @@
       then
          printprocesslog "WARN "$file" not yet closed."
-         #fileaccessed=`ssh newdaq "find $file -amin -30"`
-         #fileaccessed=`ssh 172.16.100.100 "find $file -amin -30"`
          fileaccessed=`find $file -amin -30`
-         #echo "fileaccessed: "$fileaccessed
          if ! [ "$fileaccessed" == "" ]
          then
@@ -135,9 +108,8 @@
       fi
 
+      source `dirname $0`/../Sourcefile.sh
       if ! ls $localfile >/dev/null 2>&1 
       then
-         #if ! rsync -av --progress newdaq:$file $file
-         #if ! rsync -av --progress 172.16.100.100:$file $file
-         if ! rsync -av --stats $file $localfile
+         if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
          then
             printprocesslog "ERROR something went wrong with rsync of "$file
@@ -146,7 +118,4 @@
          fi
          printprocesslog "INFO "$file" rsynced successfully."
-         rerun="no"
-      else
-         rerun="yes"
       fi
 
@@ -157,11 +126,14 @@
          runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
          roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
-         printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi
+         numevts=`$factpath/fitsdump -h $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
+         printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi" numevts "$numevts
          if [ "$runtype" == "drs-time-upshifted" ]
          then
+            printprocesslog "INFO file "$file" has runtype drs-time-upshifted -> continue "
             continue
          fi
          if [ "$runtype" == "data" ]
          then
+            numdataruns=`echo " $numdataruns + 1 " | bc -l`
             if [ "$fileerror" = "yes" ]
             then
@@ -210,4 +182,15 @@
             then
                printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
+            fi
+            callistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
+            
+            echo "cal: "$callistocount" numdat: "$numdataruns" numcallog: "${#callistofiles[@]}
+            # do not overload system in case of a lot of files to be processed
+            #  callistocount is set in setup.fact.lp.data
+            if [ $callistocount -ge $numcallistos ]
+            then
+               printprocesslog "INFO "$callistocount" RunCallisto.sh are running -> continue"
+               echo "INFO "$callistocount" RunCallisto.sh are running -> continue"
+               continue
             fi
             callistolog=`dirname $seqfile | sed -e "s/sequences/${sourcekey}\/callisto/"`"/"$night"_"$runnum"-calibration.log"
@@ -222,12 +205,14 @@
                echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
                `dirname $0`/RunCallisto.sh $sourcekey $seqfile &
-               # do not overload system in case of a lot of files to be processed
-               if [ "$rerun" = "yes" ]
-               then
-                  sleep 600
-                  printprocesslog "INFO rerun=yes -> sleep 600..."
-               fi
-            fi
-            continue
+            fi
+            continue
+         else
+            # skip a non-data run when it has not 1000 evts 
+            #   as this means probably an fad-loss
+            if [ $numevts -ne 1000 ]
+            then
+               printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$nmevts")"
+               continue
+            fi
          fi
          printprocesslog "DEBUG runrow "$runrow" (from variable) "
@@ -271,21 +256,21 @@
       fi
    done
+   printprocesslog "INFO status after loop: "$callistocount" callistos running, "$numdataruns" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
    
    # get new file lists
    printprocesslog "INFO get new file lists for "$datepath
-   #files=( `ssh newdaq "find $rawpath -type f | sort"` )
-   #files=( `ssh 172.16.100.100 "find $rawpath -type f | sort"` )
    files=( `find $rawpathnewdaq -type f | sort` )
    fileslocal=( `find $rawpath -type f | sort` )
+   callistofiles=( `find $anapath -type f -name $date*-calibration.log | sort` )
    
-#   update=
-#   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
-#   then 
-#      sleep 60
-#      echo "sleep 60..."
-#      #files=( `ssh newdaq "find $rawpath -type f | sort"` )
-#      files=( `ssh 172.16.100.100 "find $rawpath -type f | sort"` )
-#      fileslocal=( `find $rawpath -type f | sort` )
-#   fi
+   update=
+   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
+   then 
+      sleep 60
+      echo "sleep 60..."
+      files=( `find $rawpathnewdaq -type f | sort` )
+      fileslocal=( `find $rawpath -type f | sort` )
+      callistofiles=( `find $anapath -type f -name $date*-calibration.log | sort` )
+   fi
 done