#!/bin/bash
#
source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"

#root=/opt/root_svn/bin/thisroot.sh
#source $root
#factpath=/home/fact/FACT++.in-run-fad-loss

#anapath=/loc_data/analysis

#rerun="yes"
#rerun="no"
# not needed anymore as the variable is set automatically 
#   depending on whether the rawfile existed locally, 
#   i.e. depending on whether the rsync in needed
# still needed in case all rawfiles have been rsynced already

# get date (before 18h there is no new data to be processed)
datepath=`date --date="-18HOUR" +%Y/%m/%d`
#datepath="2013/03/24"
printprocesslog "INFO processing "$datepath
night=`echo $datepath | sed -e 's/\///g'`

auxpathnewdaq=/newdaq/aux/$datepath
# create aux directory on daq, if not yet there
auxpath=/loc_data/aux/$datepath
makedir $auxpath
# create path for info files needed for analysis
infopath=$anapath/info/$datepath
makedir $infopath
echo "" > $infopath/runrow.txt
# create path for sequence files
seqpath=$anapath/sequences/$datepath
makedir $seqpath
rawpathnewdaq=/newdaq/raw/$datepath
rawpath=/loc_data/raw/$datepath

# needed auxiliary files: 
#   drive file with information about current source position
drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
#   drive file with information about tracking position
drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
#   file with magic weather information 
mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
#   file with trigger rates
ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
#   file with trigger rates
tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
#   file with trigger rates
humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits

function rsync_aux_file()
{
   if ls $1
   then 
      printprocesslog "INFO rsync "$1
      #if ! rsync -av --stats $1 $2
      if ! rsync -av $1 $2
      then 
         printprocesslog "WARN rsync of "$1" failed."
      fi
   fi
}

printprocesslog "INFO get lists of raw files on newdaq and daq"
#files=( `ssh newdaq "find $rawpath -type f 2>/dev/null | sort"` )
#files=( `ssh 172.16.100.100 "find $rawpath -type f 2>/dev/null | sort"` )
files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
if [ ${#files[@]} -eq 0 ]
then
   printprocesslog "INFO no raw files available yet for "$datepath
   finish
fi
fileslocal=( `find $rawpath -type f | sort` )

# create raw directory on daq, if not yet there
makedir $rawpath


while [ ${#fileslocal[@]} -lt ${#files[@]} ] || [ "$rerun" = "yes" ]
do
   ##if ssh newdaq "ls $drivefile"
   #if ls $drivefilenewdaq
   #then 
   #   printprocesslog "INFO rsync "$drivefilenewdaq
   #   #if ! rsync -av --progress newdaq:$drivefile $drivefile
   #   #if ! rsync -av --progress 172.16.100.100:$drivefile $drivefile
   #   if ! rsync -av --stats $drivefilenewdaq $drivefile
   #   then 
   #      printprocesslog "WARN rsync of "$drivefilenewdaq" failed."
   #   fi
   #fi
   rsync_aux_file $drivefilenewdaq $drivefile
   
   # files on newdaq
   for file in ${files[@]}
   do
      printprocesslog "processing "$file
      localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
      echo "processing "$file" "$localfile

      if [ "`echo $file | grep -o drs`" == "drs" ]
      then
         nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
         #tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
         #tstop=`ssh 172.16.100.100 "source $root ; $factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
      else
         #tstop=`ssh newdaq "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
         #tstop=`ssh 172.16.100.100 "source $root ; $factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'"`
         tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
      fi
      if [ "$tstop" == "0" ]
      then
         printprocesslog "WARN "$file" not yet closed."
         #fileaccessed=`ssh newdaq "find $file -amin -30"`
         #fileaccessed=`ssh 172.16.100.100 "find $file -amin -30"`
         fileaccessed=`find $file -amin -30`
         #echo "fileaccessed: "$fileaccessed
         if ! [ "$fileaccessed" == "" ]
         then
            printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
            continue
         else
            printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
            fileerror="yes"
         fi
      fi

      if ! ls $localfile >/dev/null 2>&1 
      then
         #if ! rsync -av --progress newdaq:$file $file
         #if ! rsync -av --progress 172.16.100.100:$file $file
         if ! rsync -av --stats $file $localfile
         then
            printprocesslog "ERROR something went wrong with rsync of "$file
            rm $localfile
            continue
         fi
         printprocesslog "INFO "$file" rsynced successfully."
         rerun="no"
      else
         rerun="yes"
      fi


      if [ "`echo $localfile | grep -o drs`" != "drs" ]
      then
         runtype=`$factpath/fitsdump -h $localfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
         runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
         roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
         printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi
         if [ "$runtype" == "drs-time-upshifted" ]
         then
            continue
         fi
         if [ "$runtype" == "data" ]
         then
            if [ "$fileerror" = "yes" ]
            then
               printprocesslog "INFO do not further process corrupted file "$localfile
               fileerror=
               continue
            fi
            seqfile=$seqpath/${night}_${runnum}.seq
            printprocesslog "INFO write data-seq "$seqfile
            echo "# written by automatic analysis in LP" >$seqfile
            echo "" >> $seqfile
            echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
            echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
            echo "" >> $seqfile
            echo "DrsSequence: "$drsseq >> $seqfile
            echo "" >> $seqfile
            echo "CalRuns: "`echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'` >> $seqfile
            echo "PedRuns: "`echo $runrow | grep -E -o '[0-9]{3}pedestal300' | sed -e 's/pedestal300//g'` >> $seqfile
            echo "DatRuns: "$runnum >> $seqfile
            echo "" >> $seqfile
            echo "DrsFiles: "$drsfile >> $seqfile
            echo "" >> $seqfile
            echo "#DrsFile: "$drsfile >> $seqfile
            echo "" >> $seqfile

            tstopi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
            tstopf=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
            tstop=${tstopi}${tstopf}
            coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop}  2>/dev/null | tail -1 2>&1` )
            if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
            then
               printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
               continue
            fi
            if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
            then
               printprocesslog "WARN coordinates "${coordinates[@]}
               continue
            fi
            printprocesslog "DEBUG coordinates "${coordinates[@]}
            query="SELECT fSourceKEY FROM scheduling.source WHERE "
            query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
            query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
            sourcekey=`sendquery`
            if [ "$sourcekey" == "" ]
            then
               printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
            fi
            callistolog=`dirname $seqfile | sed -e "s/sequences/${sourcekey}\/callisto/"`"/"$night"_"$runnum"-calibration.log"
            if ! [ -e $callistolog ]
            then
               rsync_aux_file $drivefilenewdaq2 $drivefile2
               rsync_aux_file $mweatherfilenewdaq $mweatherfile
               rsync_aux_file $ratesfilenewdaq $ratesfile
               rsync_aux_file $tempfilenewdaq $tempfile
               rsync_aux_file $humfilenewdaq $humfile
               printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
               echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
               `dirname $0`/RunCallisto.sh $sourcekey $seqfile &
               # do not overload system in case of a lot of files to be processed
               if [ "$rerun" = "yes" ]
               then
                  sleep 600
                  printprocesslog "INFO rerun=yes -> sleep 600..."
               fi
            fi
            continue
         fi
         printprocesslog "DEBUG runrow "$runrow" (from variable) "
         runrow=`cat $infopath/runrow.txt`
         printprocesslog "DEBUG runrow "$runrow" (from file) "
         runrow=$runrow$runnum$runtype$roi"_"
         echo $runrow > $infopath/runrow.txt
         if echo $runrow | grep -E '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'
         then
            runrow2=`echo $runrow | grep -E -o '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'`
            run1=`echo $runrow2 | cut -d_ -f1 | sed -e 's/drs-pedestal1024//g'`
            run2=`echo $runrow2 | cut -d_ -f2 | sed -e 's/drs-gain1024//g'`
            run3=`echo $runrow2 | cut -d_ -f3 | sed -e 's/drs-pedestal1024//g'`
            run4=`echo $runrow2 | cut -d_ -f4 | sed -e 's/drs-pedestal1024//g'`
            run5=`echo $runrow2 | cut -d_ -f5 | sed -e 's/drs-time1024//g'`
            run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
            run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
            seqfile=$seqpath/${night}_${run1}.drs.seq
            printprocesslog "INFO write drs-seq "$seqfile
            echo "# written by automatic analysis in LP" > $seqfile
            echo "" >> $seqfile
            echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
            echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
            echo "" >> $seqfile
            echo "CalRuns:  "$run2 >> $seqfile
            echo "PedRuns:  "$run6" "$run7 >> $seqfile
            echo "DatRuns:  "$run5 >> $seqfile
            echo "DrsRuns:  "$run1" "$run3" "$run4 >> $seqfile
            echo "DrsFiles: "$run3" "$run6 >> $seqfile
            echo "" >> $seqfile
            echo "#DrsFile: "$run6 >> $seqfile
            echo "" >> $seqfile
            echo "" > $infopath/runrow.txt
            drsseq=$run1
            drsfile=$run6
         fi
         if echo $runrow | grep -E '[0-9]{3}pedestal300_[0-9]{3}light-pulser-ext300_'
         then
            echo "" > $infopath/runrow.txt
         fi
      fi
   done
   
   # get new file lists
   printprocesslog "INFO get new file lists for "$datepath
   #files=( `ssh newdaq "find $rawpath -type f | sort"` )
   #files=( `ssh 172.16.100.100 "find $rawpath -type f | sort"` )
   files=( `find $rawpathnewdaq -type f | sort` )
   fileslocal=( `find $rawpath -type f | sort` )
   
#   update=
#   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
#   then 
#      sleep 60
#      echo "sleep 60..."
#      #files=( `ssh newdaq "find $rawpath -type f | sort"` )
#      files=( `ssh 172.16.100.100 "find $rawpath -type f | sort"` )
#      fileslocal=( `find $rawpath -type f | sort` )
#   fi
done