Index: trunk/DataCheck/QuickLook/Step1.sh
===================================================================
--- trunk/DataCheck/QuickLook/Step1.sh	(revision 15305)
+++ trunk/DataCheck/QuickLook/Step1.sh	(revision 15305)
@@ -0,0 +1,294 @@
+#!/bin/bash
+#
+source `dirname $0`/../Sourcefile.sh
+printprocesslog "INFO starting $0"
+
+#root=/opt/root_svn/bin/thisroot.sh
+#source $root
+#factpath=/home/fact/FACT++.in-run-fad-loss
+
+#anapath=/loc_data/analysis
+
+#rerun="yes"
+#rerun="no"
+# not needed anymore as the variable is set automatically 
+#   depending on whether the rawfile existed locally, 
+#   i.e. depending on whether the rsync in needed
+# still needed in case all rawfiles have been rsynced already
+
+# get date (before 18h there is no new data to be processed)
+datepath=`date --date="-18HOUR" +%Y/%m/%d`
+#datepath="2013/03/24"
+printprocesslog "INFO processing "$datepath
+night=`echo $datepath | sed -e 's/\///g'`
+
+auxpathnewdaq=/newdaq/aux/$datepath
+# create aux directory on daq, if not yet there
+auxpath=/loc_data/aux/$datepath
+makedir $auxpath
+# create path for info files needed for analysis
+infopath=$anapath/info/$datepath
+makedir $infopath
+echo "" > $infopath/runrow.txt
+# create path for sequence files
+seqpath=$anapath/sequences/$datepath
+makedir $seqpath
+rawpathnewdaq=/newdaq/raw/$datepath
+rawpath=/loc_data/raw/$datepath
+
+# needed auxiliary files: 
+#   drive file with information about current source position
+drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
+drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
+#   drive file with information about tracking position
+drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
+drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
+#   file with magic weather information 
+mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
+mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
+#   file with trigger rates
+ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
+ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
+#   file with trigger rates
+tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
+tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
+#   file with trigger rates
+humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
+humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits
+
+function rsync_aux_file()
+{
+   if ls $1
+   then 
+      printprocesslog "INFO rsync "$1
+      #if ! rsync -av --stats $1 $2
+      if ! rsync -av $1 $2
+      then 
+         printprocesslog "WARN rsync of "$1" failed."
+      fi
+   fi
+}
+
+printprocesslog "INFO get lists of raw files on newdaq and daq"
+#files=( `ssh newdaq "find $rawpath -type f 2>/dev/null | sort"` )
+#files=( `ssh 172.16.100.100 "find $rawpath -type f 2>/dev/null | sort"` )
+files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
+if [ ${#files[@]} -eq 0 ]
+then
+   printprocesslog "INFO no raw files available yet for "$datepath
+   finish
+fi
+fileslocal=( `find $rawpath -type f | sort` )
+
+# create raw directory on daq, if not yet there
+makedir $rawpath
+
+
+while [ ${#fileslocal[@]} -lt ${#files[@]} ] || [ "$rerun" = "yes" ]
+do
+   ##if ssh newdaq "ls $drivefile"
+   #if ls $drivefilenewdaq
+   #then 
+   #   printprocesslog "INFO rsync "$drivefilenewdaq
+   #   #if ! rsync -av --progress newdaq:$drivefile $drivefile
+   #   #if ! rsync -av --progress 172.16.100.100:$drivefile $drivefile
+   #   if ! rsync -av --stats $drivefilenewdaq $drivefile
+   #   then 
+   #      printprocesslog "WARN rsync of "$drivefilenewdaq" failed."
+   #   fi
+   #fi
+   rsync_aux_file $drivefilenewdaq $drivefile
+   
+   # files on newdaq
+   for file in ${files[@]}
+   do
+      printprocesslog "processing "$file
+      localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
+      echo "processing "$file" "$localfile
+
+      if [ "`echo $file | grep -o drs`" == "drs" ]
+      then
+         nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
+         #tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
+         #tstop=`ssh 172.16.100.100 "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
+         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
+      else
+         #tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
+         #tstop=`ssh 172.16.100.100 "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
+         tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
+      fi
+      if [ "$tstop" == "0" ]
+      then
+         printprocesslog "WARN "$file" not yet closed."
+         #fileaccessed=`ssh newdaq "find $file -amin -30"`
+         #fileaccessed=`ssh 172.16.100.100 "find $file -amin -30"`
+         fileaccessed=`find $file -amin -30`
+         #echo "fileaccessed: "$fileaccessed
+         if ! [ "$fileaccessed" == "" ]
+         then
+            printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
+            continue
+         else
+            printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
+            fileerror="yes"
+         fi
+      fi
+
+      if ! ls $localfile >/dev/null 2>&1 
+      then
+         #if ! rsync -av --progress newdaq:$file $file
+         #if ! rsync -av --progress 172.16.100.100:$file $file
+         if ! rsync -av --stats $file $localfile
+         then
+            printprocesslog "ERROR something went wrong with rsync of "$file
+            rm $localfile
+            continue
+         fi
+         printprocesslog "INFO "$file" rsynced successfully."
+         rerun="no"
+      else
+         rerun="yes"
+      fi
+
+
+      if [ "`echo $localfile | grep -o drs`" != "drs" ]
+      then
+         runtype=`$factpath/fitsdump -h $localfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
+         runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
+         roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
+         printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi
+         if [ "$runtype" == "drs-time-upshifted" ]
+         then
+            continue
+         fi
+         if [ "$runtype" == "data" ]
+         then
+            if [ "$fileerror" = "yes" ]
+            then
+               printprocesslog "INFO do not further process corrupted file "$localfile
+               fileerror=
+               continue
+            fi
+            seqfile=$seqpath/${night}_${runnum}.seq
+            printprocesslog "INFO write data-seq "$seqfile
+            echo "# written by automatic analysis in LP" >$seqfile
+            echo "" >> $seqfile
+            echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
+            echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
+            echo "" >> $seqfile
+            echo "DrsSequence: "$drsseq >> $seqfile
+            echo "" >> $seqfile
+            echo "CalRuns: "`echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'` >> $seqfile
+            echo "PedRuns: "`echo $runrow | grep -E -o '[0-9]{3}pedestal300' | sed -e 's/pedestal300//g'` >> $seqfile
+            echo "DatRuns: "$runnum >> $seqfile
+            echo "" >> $seqfile
+            echo "DrsFiles: "$drsfile >> $seqfile
+            echo "" >> $seqfile
+            echo "#DrsFile: "$drsfile >> $seqfile
+            echo "" >> $seqfile
+
+            tstopi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
+            tstopf=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
+            tstop=${tstopi}${tstopf}
+            coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop}  2>/dev/null | tail -1 2>&1` )
+            if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
+            then
+               printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
+               continue
+            fi
+            if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
+            then
+               printprocesslog "WARN coordinates "${coordinates[@]}
+               continue
+            fi
+            printprocesslog "DEBUG coordinates "${coordinates[@]}
+            query="SELECT fSourceKEY FROM scheduling.source WHERE "
+            query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
+            query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
+            sourcekey=`sendquery`
+            if [ "$sourcekey" == "" ]
+            then
+               printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
+            fi
+            callistolog=`dirname $seqfile | sed -e "s/sequences/${sourcekey}\/callisto/"`"/"$night"_"$runnum"-calibration.log"
+            if ! [ -e $callistolog ]
+            then
+               rsync_aux_file $drivefilenewdaq2 $drivefile2
+               rsync_aux_file $mweatherfilenewdaq $mweatherfile
+               rsync_aux_file $ratesfilenewdaq $ratesfile
+               rsync_aux_file $tempfilenewdaq $tempfile
+               rsync_aux_file $humfilenewdaq $humfile
+               printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
+               echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
+               `dirname $0`/RunCallisto.sh $sourcekey $seqfile &
+               # do not overload system in case of a lot of files to be processed
+               if [ "$rerun" = "yes" ]
+               then
+                  sleep 600
+                  printprocesslog "INFO rerun=yes -> sleep 600..."
+               fi
+            fi
+            continue
+         fi
+         printprocesslog "DEBUG runrow "$runrow" (from variable) "
+         runrow=`cat $infopath/runrow.txt`
+         printprocesslog "DEBUG runrow "$runrow" (from file) "
+         runrow=$runrow$runnum$runtype$roi"_"
+         echo $runrow > $infopath/runrow.txt
+         if echo $runrow | grep -E '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'
+         then
+            runrow2=`echo $runrow | grep -E -o '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'`
+            run1=`echo $runrow2 | cut -d_ -f1 | sed -e 's/drs-pedestal1024//g'`
+            run2=`echo $runrow2 | cut -d_ -f2 | sed -e 's/drs-gain1024//g'`
+            run3=`echo $runrow2 | cut -d_ -f3 | sed -e 's/drs-pedestal1024//g'`
+            run4=`echo $runrow2 | cut -d_ -f4 | sed -e 's/drs-pedestal1024//g'`
+            run5=`echo $runrow2 | cut -d_ -f5 | sed -e 's/drs-time1024//g'`
+            run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
+            run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
+            seqfile=$seqpath/${night}_${run1}.drs.seq
+            printprocesslog "INFO write drs-seq "$seqfile
+            echo "# written by automatic analysis in LP" > $seqfile
+            echo "" >> $seqfile
+            echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
+            echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
+            echo "" >> $seqfile
+            echo "CalRuns:  "$run2 >> $seqfile
+            echo "PedRuns:  "$run6" "$run7 >> $seqfile
+            echo "DatRuns:  "$run5 >> $seqfile
+            echo "DrsRuns:  "$run1" "$run3" "$run4 >> $seqfile
+            echo "DrsFiles: "$run3" "$run6 >> $seqfile
+            echo "" >> $seqfile
+            echo "#DrsFile: "$run6 >> $seqfile
+            echo "" >> $seqfile
+            echo "" > $infopath/runrow.txt
+            drsseq=$run1
+            drsfile=$run6
+         fi
+         if echo $runrow | grep -E '[0-9]{3}pedestal300_[0-9]{3}light-pulser-ext300_'
+         then
+            echo "" > $infopath/runrow.txt
+         fi
+      fi
+   done
+   
+   # get new file lists
+   printprocesslog "INFO get new file lists for "$datepath
+   #files=( `ssh newdaq "find $rawpath -type f | sort"` )
+   #files=( `ssh 172.16.100.100 "find $rawpath -type f | sort"` )
+   files=( `find $rawpathnewdaq -type f | sort` )
+   fileslocal=( `find $rawpath -type f | sort` )
+   
+#   update=
+#   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
+#   then 
+#      sleep 60
+#      echo "sleep 60..."
+#      #files=( `ssh newdaq "find $rawpath -type f | sort"` )
+#      files=( `ssh 172.16.100.100 "find $rawpath -type f | sort"` )
+#      fileslocal=( `find $rawpath -type f | sort` )
+#   fi
+done
+
+
+
+