#!/bin/bash
#

# remarks: 
# rsync-server still used
# move fileerror check to main-loop?

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"

# get date (before 18h there is no new data to be processed)
if [ "$certaindate" != "" ]
then
   checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
   if [ "$checkstring" = "" ]
   then
      echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
      finish
   fi
   datepath=$certaindate
else
   datepath=`date --date="-19HOUR" +%Y/%m/%d`
fi
date=`echo $datepath | sed -e 's/\///g'`
printprocesslog "INFO processing "$datepath

auxpathnewdaq=/newdaq/aux/$datepath
#auxpathnewdaq=/loc_data/aux/$datepath #to run on newdaq
# create aux directory on daq, if not yet there
auxpath=/loc_data/aux/$datepath
makedir $auxpath >/dev/null
# create path for info files needed for analysis
infopath=$anapath/info/$datepath
makedir $infopath >/dev/null
echo "" > $infopath/runrow.txt
# create path for callisto output
calpath=$anapath/callisto/$datepath
makedir $calpath >/dev/null
rawpathnewdaq=/newdaq/raw/$datepath
#rawpathnewdaq=/loc_data/raw/$datepath #to run on newdaq
rawpath=/loc_data/raw/$datepath

# needed auxiliary files: 
#   drive file with information about current source position
drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
drivefilenewdaq=$auxpathnewdaq/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
#   drive file with information tracking position
drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
drivefilenewdaq2=$auxpathnewdaq/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
#   file with magic weather information 
mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
mweatherfilenewdaq=$auxpathnewdaq/${date}.MAGIC_WEATHER_DATA.fits
#   file with trigger rates
ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
ratesfilenewdaq=$auxpathnewdaq/${date}.FTM_CONTROL_TRIGGER_RATES.fits
#   file with trigger rates
tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
tempfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_TEMPERATURE.fits
#   file with trigger rates
humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
humfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_HUMIDITY.fits

function rsync_aux_file()
{
   if ls $1 >/dev/null 2>&1
   then 
      printprocesslog "INFO rsync "$1
      # rsync 
      #   from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/
      #   to daq (/daq = /loc_data on daq)
      rsyncservernewdaq=`echo $1 | sed -e 's/^\//172.16.100.100::/'` 
      # old
      if ! rsync -a -T $rsynctempdir $1 $2
      # new (workaround for problems on daq)
      #if ! rsync -a -T $rsynctempdir $rsyncservernewdaq $2
      then 
         printprocesslog "WARN rsync of "$1" failed."
      fi
   else
      printprocesslog "WARN "$1" missing."
   fi
}

function check_daq()
{
   diskusage=( `df -P /raid10 | grep raid10 ` )
   # check if more than 700 GB are left on /loc_data
   if [ ${diskusage[3]} -lt $disklimitdaq ] 
   then
      echo "WARN less than 700 left on /raid10 on node "$HOSTNAME
      printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME
      df -h /raid10
      finish
   fi
}

check_daq

printprocesslog "INFO get lists of raw files on newdaq and daq"
files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
# to treat links use:
#files=( `find -L $rawpathnewdaq -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )

if [ ${#files[@]} -eq 0 ]
then
   printprocesslog "INFO no raw files available yet for "$datepath
   finish
fi
fileslocal=( `find -L $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null| sort` )

callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null| sort` )
# get number of dataruns from DB
query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
numdataruns=`sendquery`
query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6"
numlpruns=`sendquery`
query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300"
numpedruns=`sendquery`
query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5"
numdrstime=`sendquery`
numpedruns=0
#numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l`
numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l`

# create raw directory on daq, if not yet there
makedir $rawpath >/dev/null

#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated

while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
do
   # only continue with script 
   #  when there is more than 10% space on daq
   source `dirname $0`/../Sourcefile.sh
   check_daq
   
   numcalibrated=0
   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
   printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated

   rsync_aux_file $drivefilenewdaq $drivefile
   rsync_aux_file $drivefilenewdaq2 $drivefile2
   
   # files on newdaq
   for file in ${files[@]}
   do
      printprocesslog "processing "$file
      #echo "processing "$file
      localfile=`echo $file | sed -e 's/newdaq/loc_data/'`

      source `dirname $0`/../Sourcefile.sh

      # check if file is already transferred
      if ! ls $localfile >/dev/null 2>&1 
      then
         # check if it is drs-file
         #   get stop time from raw-file
         if [ "`echo $file | grep -o drs`" == "drs" ]
         then
            nondrs=`basename $file | sed -e 's/[.]drs//g'`
            nondrsfile=`find -L $rawpath -name $nondrs.*z`
            tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
         else
            tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
         fi
         # when stop time is 0, file is not closed
         # when an error is returned the tstop is empty
         if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
         then
            printprocesslog "WARN "$file" not yet closed."
            # if a file is not closed and not touched for 30 minutes, 
            #    it is assumed corrupted and still transferred
            fileaccessed=`find $file -amin -30`
            if ! [ "$fileaccessed" == "" ]
            then
               printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
               continue
            else
               printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
               fileerror="yes"
            fi
         fi

         # rsync 
         #   from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/
         #   to daq (/daq = /loc_data on daq)
         # to access rsync server via the dedicated network between 
         #    daq and newdaq, use 172.16.100.100::newdaq
         filersyncserver=`echo $file | sed -e 's/^\//172.16.100.100::/'` 
         # old
         ##if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
         if ! rsync -a -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
         # new
         #if ! rsync -a -W -T $rsynctempdir --bwlimit=$bwlimit $filersyncserver $localfile
         then
            printprocesslog "ERROR something went wrong with rsync of "$file
            rm $localfile
            continue
         fi
         printprocesslog "INFO "$file" rsynced successfully."
      fi

      # for .drs.fits files no further treatment needed
      if [ "`echo $localfile | grep -o drs`" == "drs" ]
      then
         continue
      fi
      
#      # temporary check (needed to run on newdaq)
#      if [ "`echo $file | grep -o drs`" == "drs" ]
#      then
#         nondrs=`basename $file | sed -e 's/[.]drs//g'`
#         nondrsfile=`find -L $rawpath -name $nondrs.*z`
#         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
#      else
#         tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
#      fi
#      # when stop time is 0, file is not closed
#      # when an error is returned the tstop is empty
#      if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
#      then
#         printprocesslog "WARN "$file" not yet closed. -> continue (temporary check)"
#         continue
#      fi
#      # end temporary check
      
      # treat other files (.fits.fz)
      runtype=`$factpath/fitsdump -h $localfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
      if [ "$runtype" != "data" ]
      then
         # skip a non-data run when it has not 1000 evts 
         #   as this means probably an fad-loss 
         #   and these runs are repeated in that case
         numevts=`$factpath/fitsdump -h $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
         if [ "$numevts" == "" ]
         then
            printprocesslog "WARN could not get number of events from file "$file" -> continue "
            #echo "WARN could not get number of events from file "$file" -> continue "
            continue
         fi
         if [ $numevts -ne 1000 ]
         then
            printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")"
            continue
         fi
      fi

      # get run number
      runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
      
      # what is needed to process the different runs? 
      #   P: run#(P), run#(drs-file)
      #   C: run#(C), run#(drs-file), run#(drs-time)
      #   D: run#(D), run#(drs-file), run#(drs-time), ? 
      # what is drs-file? pedestal, roi300, has drs.fits
      callistolog=$calpath"/"$date"_"$runnum"-calibration.log"
      case $runtype in
         data) # treat D-runs
            if [ "$fileerror" = "yes" ]
            then
               printprocesslog "INFO do not further process corrupted file "$localfile
               fileerror=
               continue
            fi
            
            # some accounting
            printprocesslog "DEBUG counting callisto logs and set data files +1."
            # get number of callisto logs
            runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
            # count runs to be calibrated
            numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
            printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]}
            
            # do not overload system in case of a lot of files to be processed
            #  numruncallistos is set in setup.fact.lp.data
            if [ $runcallistocount -ge $numruncallistos ]
            then
               printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue"
               continue
            fi
            
            # starting calibration
            if ! [ -e $callistolog ]
            then
               rsync_aux_file $drivefilenewdaq $drivefile
               rsync_aux_file $drivefilenewdaq2 $drivefile2
               rsync_aux_file $mweatherfilenewdaq $mweatherfile
               rsync_aux_file $ratesfilenewdaq $ratesfile
               rsync_aux_file $tempfilenewdaq $tempfile
               rsync_aux_file $humfilenewdaq $humfile
               
               calfile=$calpath"/"$date"_"$runnum"_C.root"
               printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" outpath "$outpath" calfile "$calfile
               `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $calpath $calfile &
            fi
            continue
            ;;
         pedestal) # treat P-runs
            roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
            if [ $roi -eq 300 ]
            then
               # check drs-file
               drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
               if [ -e $drsfile ]
               then
                  # set name of drs-file
                  drscalib=$drsfile
                  continue
               #else
                  # not needed for QLA
                  #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
                  #if ! [ -e $callistolog ]
                  #then
                  #   pedfile=$calpath"/"$date"_"$runnum"-pedestal.root"
                  #   # count runs to be calibrated
                  #   printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
                  #   echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
                  #  `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile &
                  #fi
               fi
            fi
            ;;
         light-pulser-ext) # treat C-runs
            # do lp-treatment -> not needed for QLA
            #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root"
            #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
            #if ! [ -e $callistolog ]
            #then
            #   if [ -e $drstime ]
            #   then
            #      # count runs to be calibrated
            #      printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
            #      echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
            #      `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile &
            #   fi
            #fi
            ;;
         drs-time) # treat drs-time runs
            # do drs-timing calibration
            drstime=$calpath"/"$date"_"$runnum"-drstime.root"
            # starting calibration
            numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
            if ! [ -e $callistolog ]
            then
               # count runs to be calibrated
               printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
               #echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
               `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime &
            fi
            ;;
         drs-pedestal) # get drs-pedestal
            roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
            drs=`$factpath/fitsdump -h $localfile  2>/dev/null | grep DRSCALIB | grep -E -o " T " `
            if [ $roi -eq 1024 ] && [ "$drs" == " T " ]
            then
               drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
            fi
            ;;
         *) # other runs
            printprocesslog "INFO file "$file" has runtype "$runtype" -> continue "
            continue
            ;;
      esac
   done
   printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
   
   # get new file lists
   printprocesslog "INFO get new file lists for "$datepath
   files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
   fileslocal=( `find -L $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
   callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` )
   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
   printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated

   # wait and get new file lists
   update=
   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
   then 
      printprocesslog "INFO wait 60 seconds."
      sleep 60
      printprocesslog "INFO get new file lists for "$datepath
      files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
      fileslocal=( `find -L $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
      callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` )
   fi
   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
   printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
   sleep 30
   printprocesslog "INFO sleep 30"
done