#!/bin/bash

#####################################
#
#	CONFIGURATION VARIABLES
#
#####################################
#temporary text files
colDescFile="coldesc.lis"
dataFile="data.lis"
keywordsFile="keywords.lis"
logfile="logfile.txt"
#date for which the script should be run
year=$1
month=$2
day=$3
#target folders
#sourceFolder="/data00/fact-construction"
sourceFolder="/archive/fact/rev_1"
targetFolder=`pwd`

#make source folder relative to target
common_part=$targetFolder
back=
while [ "$common_part" != "/" ] && [ "${sourceFolder#$common_part}" == "$sourceFolder" ] 
do
  common_part=`dirname $common_part`
  back="../${back}"
#  echo $common_part $back
done
baseFolder=$back${sourceFolder#$common_part}

#remove useless artifacts from baseFolder
pathChanged="true"
while [ "$pathChanged" == "true" ]
do
	pathChanged="false"
	newBaseFolder=`echo $baseFolder | sed -e 's/\/\//\//g'`
	if [ "$newBaseFolder" != "$baseFolder" ]
	then
		pathChanged="true"
		echo "$baseFolder >>> $newBaseFolder"
	fi
	baseFolder=$newBaseFolder
done

rawFolder=$baseFolder"/raw/"$year"/"$month"/"$day
auxFolder=$baseFolder"/aux/"$year"/"$month"/"$day
tempFile="./tempFile.txt"
tempFits="./tempFits.fits"
#input aux files
trackingTable="DRIVE_CONTROL_TRACKING_POSITION"
trackingFile=$auxFolder"/"$year$month$day"."$trackingTable".fits"
triggerTable="FTM_CONTROL_TRIGGER_RATES"
triggerFile=$auxFolder"/"$year$month$day"."$triggerTable".fits"
staticTable="FTM_CONTROL_STATIC_DATA"
staticFile=$auxFolder"/"$year$month$day"."$staticTable".fits"
voltagesTable="BIAS_CONTROL_VOLTAGE"
voltagesFile=$auxFolder"/"$year$month$day"."$voltagesTable".fits"
currentsTable="BIAS_CONTROL_CURRENT"
currentFile=$auxFolder"/"$year$month$day"."$currentsTable".fits"

basePathAux=$auxFolder"/"$year$month$day

auxTable="BIAS_CONTROL_NOMINAL"
auxDesc="Some_Description"
auxFile=$basePathAux"."$auxTable".fits"

###########################################
#
#	WRITE EXTRA AUX FILE
#	Writes an extra entry to the data file
#	Also checks for existence of said file
#
##########################################
function writeExtraAuxFile() {

	auxFile=$basePathAux"."$auxTable".fits"
	if ! [ -e $auxFile ]
	then
    	     dummy=3
	     #echo "WARNING: Optionnal aux file "$auxFile" not found." | tee -a $logfile 2>&1
	else
	     echo $auxTable" BINTABLE URL ../../../"$auxFile" 1 1 "$auxDesc >> $dataFile	
	fi
}

currentEntry=""
roi=-1
roiTM=-1
runnumber=-1
doNotDoThisEntry="false"
###########################################
#
#	WRITE HEADER KEYS TO INPUT FILE
#	Extract and write the header keywords to the above devined output text files. 
#	These text files are then meant to feed the fits creation tool fcreate
#	Large parts of this function code were retaken from Daniela's scripts
#
###########################################
function writeHeaderKeysToInputFile () {
#echo "Writing header keys for "$currentEntry
	#get all the missing data (everything appart from roi and roiTM)
	runtype=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
        numevents=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
        numphys=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
        numext1=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGEXT1' | grep -E -o "'[0-9]+'" | grep -E -o '[0-9]+'`
        numext2=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGEXT2' | grep -E -o "'[0-9]+'" | grep -E -o '[0-9]+'`
        numelp=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
        numilp=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
        numoth=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
        numped=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
        numtime=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
	dateRaw=$year$month$day
	
	if [ "$runtype" == "" ]
	then
		runtype="N/A"
	fi
	if [ "$numevents" == "" ]
	then
		numevents="0"
	fi
	
#	dateRaw=`echo $currentEntry | grep -E -o '20[1-3][0-9][01][0-9][012][0-9]'`
#echo "dateRaw: "$dateRaw" yearmonthday: "$year$month$day

         # in newest data start time is in DATE-OBS
         # in older data start time is in TSTART
         # in the beginning TSTART was empty
        runstart=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
        runstart2=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
        if [ "$runstart" == ""  ]
        then
           if [ "$runstart2" == ""  ]
           then
              runstart=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
           else
              runstart=$runstart2
           fi
        fi
        # in newest data start time is in DATE-END
        # in older data start time is in TSTOP
        # in the beginning TSTOP was empty
        runstop=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
        runstop2=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
        if [ "$runstop" == ""  ]
        then
           if [ "$runstop2" == ""  ]
           then
             runstop=`stat $currentEntry  2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
           else
              runstop=$runstop2
           fi
        fi

      # set runtype to 'unknown' if no runtype could be queried
      if [ "$runtype" == "" ]
      then
         runtype="n/a"
      fi
      # on 15.11.2011 the runtypes had different names
      if [ "$date" == "2011/11/15" ]
      then
         if [ "$runtype" == "drs-calib" ]
         then
            runtype="drs-gain"
         fi
         if [ "$runtype" == "drs-time-calib" ]
         then
            runtype="drs-time"
         fi
         if [ "$runtype" == "pedestal" ]
         then
            runtype="drs-pedestal"
         fi
         if [ "$runtype" == "light-pulser" ]
         then
            runtype="light-pulser-ext"
         fi
         if [ "$runtype" == "pedestal-on" ]
         then
            runtype="pedestal"
         fi
      fi
	#now take care of the slow control data.
	#first get the start and stop time of the raw file

      mjdref=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
      tstarti=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
      tstartf=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
      tstopi=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
      tstopf=`/opt/FACT++/fitsdump -h -t Events $currentEntry  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
      doNotDoThisEntry="false"
      if [ "$tstarti" == "0" ] || [ "$tstopi" == "0" ] || [ "$mjdref" == "0" ]
      then
         echo "ERROR: "$currentEntry" has tstart, tstop or mjdref = 0"
      fi
	
      if [ "$tstartf" == "" ]
      then
      	doNotDoThisEntry="true"
	echo $currentEntry >> problemWithTSTART.txt
	echo "WARNING: "$currentEntry" Has problems with Dates"
      fi
      if [ "$tstarti" == "" ]
      then
      		tstarti="0"
      fi
      if [ "$tstopi" == "" ]
      then
      		tstopi="0"
      fi
      if [ "$mjdref" == "" ]
      then
      		mjdref="0"
	fi
      if [ "$tstartf" == "" ]
      then
      		tstartf="0"
      fi
      if [ "$tstopf" == "" ]
      then
		tstopf="0"
      fi

      tstart=`echo " $tstarti + $mjdref + $tstartf " | bc -l`
      tstop=`echo " $tstopi + $mjdref + $tstopf " | bc -l`
      exposure=`echo "$tstop - $tstart " | bc -l`
      exposure=`echo "$exposure * 86400" | bc -l `

      #now get relevant data from daily files
      #first get mjdref for the aux file and adapt start time accordingly
      auxmjdref=`/opt/FACT++/fitsdump -h -t $trackingTable $trackingFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
      if [ "$auxmjdref" == "" ]
      then
      	echo "ERROR: "$trackingFile" has no mjdref available. Aborting script"
	exit
      fi

      tstartaux=`echo " $tstart - $auxmjdref " | bc -l` 
      tstopaux=`echo " $tstop - $auxmjdref " | bc -l`    
      ftcopy $trackingFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col Ra;Dec;Zd;Az;Time]' history=NO !$tempFits
      /opt/FACT++/fitsdump $tempFits -c Ra --stat -o $tempFile 2>/dev/null
      ramin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      ramax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      ramean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      if [ "$ramin" != "" ]
      then
	      ramin=`echo "$ramin * 15" | bc -l`
      		ramax=`echo "$ramax * 15" | bc -l`
      		ramean=`echo "$ramean * 15" | bc -l`
	fi
      /opt/FACT++/fitsdump $tempFits -c Dec --stat -o $tempFile 2>/dev/null
      decmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      decmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      decmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      /opt/FACT++/fitsdump $tempFits -c Zd --stat -o $tempFile 2>/dev/null
      zdmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      zdmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      zdmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      /opt/FACT++/fitsdump $tempFits -c Az --stat -o $tempFile 2>/dev/null
      azmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      azmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      azmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`

      auxmjdref=`/opt/FACT++/fitsdump -h -t $triggerTable $triggerFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
      if [ "$auxmjdref" == "" ]
      then
      	echo "ERROR: "$triggerFile" has no mjdref available. Aborting script"
	exit
      fi
      tstartaux=`echo " $tstart - $auxmjdref " | bc -l` 
      tstopaux=`echo " $tstop - $auxmjdref " | bc -l`        
      ftcopy $triggerFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col TriggerRate;Time]' history=NO !$tempFits
      /opt/FACT++/fitsdump $tempFits -c TriggerRate --stat -o $tempFile 2>/dev/null
      ratemin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      ratemax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      ratemean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
      ratemedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` 

      auxmjdref=`/opt/FACT++/fitsdump -h -t $staticTable $staticFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
      if [ "$auxmjdref" == "" ]
      then
      	echo "ERROR: "$staticFile" has no mjdref available. Aborting script"
	exit
      fi
      iter=0
      timeShift=0
      threshmin=""
      while [ "$threshmin" == "" ] && [ "$iter" != "10" ]
      do
         tstartaux=`echo " $tstart - $auxmjdref - $timeShift " | bc -l` 
         tstopaux=`echo " $tstop - $auxmjdref " | bc -l`       
	 ftcopy $staticFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col PatchThresh;Time]' history=NO !$tempFits 
         /opt/FACT++/fitsdump $tempFits -c PatchThresh --stat -o $tempFile 2>/dev/null
	 threshmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         threshmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         threshmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         threshmedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` 

	 timeShift=`echo " $timeShift + 0.00011574 " | bc -l` 
	 iter=`echo " $iter + 1 " | bc -l`
      done

      #########################################################################################################################################################
      auxmjdref=`/opt/FACT++/fitsdump -h -t $voltagesTable $voltagesFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
      if [ "$auxmjdref" == "" ]
      then
      	echo "ERROR: "$voltagesFile" has no mjdref available. Aborting script"
	exit
      fi
      iter=0
      timeShift=0
      biasvoltmin=""
      while [ "$biasvoltmin" == "" ] && [ "$iter" != "10" ]
      do
         tstartaux=`echo " $tstart - $auxmjdref " | bc -l` 
         tstopaux=`echo " $tstop - $auxmjdref " | bc -l`   
         ftcopy $voltagesFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col U;Time]' !$tempFits
         /opt/FACT++/fitsdump $tempFits -c U[0:319] --stat -o $tempFile 2>/dev/null
         biasvoltmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         biasvoltmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         biasvoltmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         biasvoltmedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` 
	 timeShift=`echo " $timeShift + 0.00011574 " | bc -l`
 	 iter=`echo " $iter + 1 " | bc -l`

      done

      auxmjdref=`/opt/FACT++/fitsdump -h -t $currentsTable $currentFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
      if [ "$auxmjdref" == "" ]
      then
      	echo "ERROR: "$currentFile" has no mjdref available. Aborting script"
	exit
      fi
      iter=0
      timeShift=0
      biascurrentmin=""
      while [ "$biascurrentmin" == "" ] && [ "$iter" != "10" ]
      do
         tstartaux=`echo " $tstart - $auxmjdref " | bc -l` 
         tstopaux=`echo " $tstop - $auxmjdref " | bc -l`       
         ftcopy $currentFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col I;Time]' !$tempFits
         /opt/FACT++/fitsdump $tempFits -c I[0:319] --stat -o $tempFile 2>/dev/null
         biascurrentmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         biascurrentmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         biascurrentmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
         biascurrentmedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` 
	 timeShift=`echo " $timeShift + 0.00011574 " | bc -l`
 	 iter=`echo " $iter + 1 " | bc -l`

#echo "biascurrent min, max. mean, median: "$biascurrentmin" "$biascurrentmax" "$biascurrentmean" "$biascurrentmedian
      done

      #write the retrieved data to the input file for creating fits afterwards
      echo "RUNTYPE "$runtype" / run type"  >> $keywordsFile	
      echo "ROI "$roi" / region of interest" >> $keywordsFile
      echo "ROITM "$roiTM" / Roi for time markers" >> $keywordsFile
      echo "NUMEVENT "$numevents" / number of events" >> $keywordsFile
      echo "NUMPHYS "$numphys" / num of physical triggers" >> $keywordsFile
      echo "NUMEXT1 "$numext1" / num of Light pulser 1 triggers" >> $keywordsFile
      echo "NUMEXT2 "$numext2" / num of Light pulser 2 triggers" >> $keywordsFile
      echo "NUMELP "$numelp" / num of external Light pulser triggers" >> $keywordsFile
      echo "NUMILP "$numilp" / num of internal Light pulser triggers" >> $keywordsFile
      echo "NUMOTH "$numoth" / num of other triggers" >> $keywordsFile
      echo "NUMPED "$numped" / num of pedestal triggers" >> $keywordsFile
      echo "NUMTIME "$numtime" / num of time calibration triggers" >> $keywordsFile
      echo "DATEOBS "$runstart" / start of observation" >> $keywordsFile
      echo "MJDREF "$mjdref" / reference of MjD values" >> $keywordsFile
      echo "EXPOSURE "$exposure" /duration of run" >> $keywordsFile
      echo "TSTARTI "$tstarti" / first event arrival time (int)" >> $keywordsFile
      echo "TSTARTF "$tstartf" / first event arrival time (frac)" >> $keywordsFile
      echo "TSTOPI "$tstopi" / last event arrival time (int)" >> $keywordsFile
      echo "TSTOPF "$tstopf" / last event arrival time (frac)" >> $keywordsFile
      echo "RAMIN "$ramin" / min value of right ascension" >> $keywordsFile
      echo "RAMAX "$ramax" / max value of right ascension" >> $keywordsFile
      echo "RAMEAN "$ramean" / mean value of right ascension" >> $keywordsFile
      echo "DECMIN "$decmin" / min value of declination" >> $keywordsFile
      echo "DECMAX "$decmax" / max value of declination" >> $keywordsFile
      echo "DECMEAN "$decmean" / mean value of declination" >> $keywordsFile
      echo "ZDMIN "$zdmin" / min value of zenith distance" >> $keywordsFile
      echo "ZDMAX "$zdmax" / max value of zenith distance" >> $keywordsFile
      echo "ZDMEAN "$zdmean" / mean value of zenith distance" >> $keywordsFile
      echo "AZMIN "$azmin" / min value of azimuth" >> $keywordsFile
      echo "AZMAX "$azmax" / max value of azimuth" >> $keywordsFile
      echo "AZMEAN "$azmean" / mean value of azimuth" >> $keywordsFile
      echo "RATEMIN "$ratemin" / min value of trigger rates" >> $keywordsFile
      echo "RATEMAX "$ratemax" / max value of trigger rates" >> $keywordsFile
      echo "RATEMEAN "$ratemean" / mean value of trigger rates" >> $keywordsFile
      echo "RATEMED "$ratemedian" / median value of trigger rates" >> $keywordsFile
      echo "THRESMIN "$threshmin" / min threshold value" >> $keywordsFile
      echo "THRESMAX "$threshmax" / max threshold value" >> $keywordsFile
      echo "THRESMEA "$threshmean" / mean threshold value" >> $keywordsFile
      echo "THRESMED "$threshmedian" / max threshold value" >> $keywordsFile
      echo "BIASVMIN "$biasvoltmin" / min bias voltage (V)" >> $keywordsFile
      echo "BIASVMAX "$biasvoltmax" / max bias voltage (V)" >> $keywordsFile
      echo "BIASVMEA "$biasvoltmean" / mean bias voltage (V)" >> $keywordsFile
      echo "BIASVMED "$biasvoltmedian" / median bias voltage (V)" >> $keywordsFile
      echo "BIASAMIN "$biascurrentmin" / min bias current (uA)" >> $keywordsFile
      echo "BIASAMAX "$biascurrentmax" / max bias current (uA)" >> $keywordsFile
      echo "BIASAMEA "$biascurrentmean" / mean bias current (uA)" >> $keywordsFile
      echo "BIASAMED "$biascurrentmedian" / median bias current (uA)" >> $keywordsFile

      
      echo "EXTNAME GROUPING / grouping table" >> $keywordsFile
      echo "GRPNAME FACT-RAW / name of group" >> $keywordsFile
      echo "RUNNUM "$runnumber" / run number" >> $keywordsFile
      echo "RUNID "$dateRaw"_"$runnumber" / Run Id" >> $keywordsFile

}

#today=`date +%F`
#alias alsoToLog='tee -a $logfile 2>&1'

#echo "" | tee -a $logfile 2>&1
#echo "" | tee -a $logfile 2>&1
#echo "" | tee -a $logfile 2>&1

#cleanup logfile
rm -f $logfile

#echo `date`" executing "$0 | tee -a $logfile 2>&1



#first of all, let's make sure that required slow control files are present
if ! [ -e $trackingFile ]
then
    echo "ERROR: Required aux file "$trackingFile" not found. Aborting script" | tee -a $logfile 2>&1
    exit
fi
if ! [ -e $triggerFile ]
then
    echo "ERROR: Required aux file "$triggerFile" not found. Aborting script" | tee -a $logfile 2>&1
    exit
fi
if ! [ -e $staticFile ]
then
    echo "ERROR: Required aux file "$staticFile" not found. Aborting script" | tee -a $logfile 2>&1
    exit
fi
if ! [ -e $voltagesFile ]
then
    echo "ERROR: Required aux file "$voltagesFile" not found. Aborting script" | tee -a $logfile 2>&1
    exit
fi
if ! [ -e $currentFile ]
then
    echo "ERROR: Required aux file "$currentFile" not found. Aborting script" | tee -a $logfile 2>&1
    exit
fi

#next define the format of the grouping file

rm -f $colDescFile
echo "MEMBER_NAME 60A"      >> $colDescFile
echo "MEMBER_XTENSION 8A"   >> $colDescFile 
echo "MEMBER_URI_TYPE 3A"   >> $colDescFile
echo "MEMBER_LOCATION 256A" >> $colDescFile
echo "MEMBER_VERSION 1J"    >> $colDescFile
echo "MEMBER_POSITION 1J"   >> $colDescFile
echo "DATA_TYPE 60A"	    >> $colDescFile

entries=`find $rawFolder -type f -name '*.fits.gz' | sort`
calibs=""
calibFiles=""
calibDrsFiles=""
calibDrsPedestalFiles=""
calibsRoi=""
calibsRoiTM=""

#data=""
#correspondingCalib=""
numCalibs=0
#numData=0

if [ "$year" == "" ]
then
	echo "Missing year argument"
	exit
fi

if [ "$month" == "" ]
then
	echo "Missing month argument"
	exit
fi

if [ "$day" == "" ]
then
	echo "Missing day argument"
	exit
fi

numEntries=0
for item in ${entries[@]}
do
    numEntries=`expr $numEntries + 1`
done

echo "There are "$numEntries" entries to examine in folder "$rawFolder | tee -a $logfile 2>&1

#echo "Will now erase entries generated previously..."
#toDelete=`find $targetFolder -type f -name '*_raw.fits' | sort`
#for delete in ${toDelete[@]}
#do
#	echo "removing "$delete
#	rm $delete
#done
#toDelete=`find $targetFolder -type f -name '*_raw.txt' | sort`
#for delete in ${toDelete[@]}
#do
#	echo "removing $delete"
#	rm $delete
#done
#echo "done"

for entry in ${entries[@]}
do
	filename=`echo $entry | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
        filedate=`echo $filename | cut -d_ -f1`
        runnumberInter=`echo $filename | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'`
        runnumber=`printf %03d $runnumberInter`
        #figure out whether this is a drs calib
	if [ "`echo $entry | grep drs`" != "" ]
        then
		#check if this is the pedestal run. for this, grep the associated raw data file and check for previous calibrations (drs pedestal and drs gain)
		minusOne=`expr $runnumber - 1`
		minusOne=`printf %03d $minusOne`
		minusTwo=`expr $runnumber - 2`
		minusTwo=`printf %03d $minusTwo`
		pedestalRaw=$rawFolder/$filename.fits.gz
		DrsGainRaw=$rawFolder/$filedate"_"$minusOne.fits.gz
		DrsPedestalRaw=$rawFolder/$filedate"_"$minusTwo.fits.gz
		DrsGain=$rawFolder/$filedate"_"$minusOne.drs.fits.gz
		DrsPedestal=$rawFolder/$filedate"_"$minusTwo.drs.fits.gz
		pedestalkey=""
		drsGainKey=""
		drsPedestalKey=""
		if [ -f $pedestalRaw ]
		then
			pedestalKey=`/opt/FACT++/fitsdump -h $pedestalRaw 2>/dev/null | grep "'pedestal'"`
		fi
		if [ "$pedestalKey" == "" ]
		then #the current drs file is NOT a pedestal. Continue
			continue
		fi
#		echo "Found Pedestal entry "$entry
		if [ -f $DrsGainRaw ] && [ -f $DrsGain ]
		then
			drsGainKey=`/opt/FACT++/fitsdump -h $DrsGainRaw 2>/dev/null | grep "'drs-gain'"`
		else
			DrsGain=""
		fi
		if [ -f $DrsPedestalRaw ] && [ -f $DrsPedestal ]
		then
			drsPedestalKey=`/opt/FACT++/fitsdump -h $DrsPedestalRaw 2>/dev/null | grep "'drs-pedestal'"`
		else
			DrsPedestal=""
		fi
		
		if [ "$drsPedestalKey" != "" ]
		then
			calibtstarti=`/opt/FACT++/fitsdump -h $DrsPedestalRaw  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
		        calibtstartf=`/opt/FACT++/fitsdump -h $DrsPedestalRaw  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
		else
			if [ "$drsGainKey" != "" ]
			then
				calibtstarti=`/opt/FACT++/fitsdump -h $DrsGainRaw  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
			        calibtstartf=`/opt/FACT++/fitsdump -h $DrsGainRaw  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
			else
				calibtstarti=`/opt/FACT++/fitsdump -h $pedestalRaw  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
		        	calibtstartf=`/opt/FACT++/fitsdump -h $pedestalRaw  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
			fi
		fi
		calibtstopi=`/opt/FACT++/fitsdump -h $pedestalRaw  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
      		calibtstopf=`/opt/FACT++/fitsdump -h $pedestalRaw  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`

		auxstarti=`/opt/FACT++/fitsdump -h $voltagesFile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
		auxstartf=`/opt/FACT++/fitsdump -h $voltagesFile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
		
		if [ "$auxstarti" == "" ]
		then
			auxstarti="0"
		fi
		if [ "$auxstartf" == "" ]
		then
			auxstartf="0"
		fi
		if [ "$calibtstarti" == "" ]
		then
			calibtstarti="0"
		fi
		if [ "$calibtstartf" == "" ]
		then
			calibtstartf="0"
		fi
		if [ "$calibtstopi" == "" ]
		then
			calibtstopi="0"
		fi
		if [ "$calibtstopf" == "" ]
		then
			calibtstopf="0"
		fi
		
		auxstart=`echo " $auxstarti + 40587 + $auxstartf " | bc -l`
		calibtstart=`echo " $calibtstarti + 40587 + $calibtstartf " | bc -l`
		calibtstop=`echo " $calibtstopi + 40587 + $calibtstopf " | bc -l`
		iter=0
      		timeShift=0
		biasvoltmean=""
         	tstartaux=`echo "  $calibtstart - 40587 - $timeShift" | bc -l` 
		compare=`echo " $tstartaux >= $auxstart " | bc -l`

      		while [ "$biasvoltmean" == "" ] && [ "$iter" != "10" ] && [ "$compare" == "1" ]
      		do
         		tstartaux=`echo "  $calibtstart - 40587 - $timeShift" | bc -l`
			compare=`echo " $tstartaux >= $auxstart " | bc -l` 
         		tstopaux=`echo " $calibtstop - 40587 " | bc -l`   
         		ftcopy $voltagesFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col U;Time]' !$tempFits
         		/opt/FACT++/fitsdump $tempFits -c U[0:319] --stat -o $tempFile 2>/dev/null
         		biasvoltmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'`
	 		timeShift=`echo " $timeShift + 0.000011574 " | bc -l`
 	 		iter=`echo " $iter + 1 " | bc -l`
		done
		auxstarti=`/opt/FACT++/fitsdump -h $currentFile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
		auxstartf=`/opt/FACT++/fitsdump -h $currentFile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
		auxstart=`echo " $auxstarti + $auxstartf " | bc -l`
		iter=0
		timeShift=0
		biascurrentmean=""
		tstartaux=`echo " $calibtstart - 40587 - $timeShift" | bc -l`
		compare=`echo " $tstartaux >= $auxstart " | bc -l`
		while [ "$biascurrentmean" == "" ] && [ "$iter" != "10" ] && [ "$compare" == "1" ]
		do
			tstartaux=`echo " $calibtstart - 40587 - $timeShift " | bc -l`
			compare=`echo " $tstartaux >= $auxstart " | bc -l` 
        		tstopaux=`echo " $calibtstop - 40587 " | bc -l`       
        		ftcopy $currentFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col I;Time]' !$tempFits
			/opt/FACT++/fitsdump $tempFits -c I[0:319] --stat -o $tempFile 2>/dev/null
         		biascurrentmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` 
			timeShift=`echo " $timeShift + 0.000011574 " | bc -l`
 			iter=`echo " $iter + 1 " | bc -l`
		done

		if [ "$biasvoltmean" == "" ]
		then
		     	biasvoltmean=0
		fi
#		echo "Bias voltage: "$biasvoltmean
		if [ "$biascurrentmean" == "" ]
		then
		     	biascurrentmean=0
		fi
		compare=`echo " $biascurrentmean < 5 " | bc -l`
		if [ "$pedestalKey" != "" ] && [ "$drsGainKey" != "" ] && [ "$drsPedestalKey" != "" ] && [ "$biasvoltmean" == "0" ] && [ "$compare" == "1" ]
		then
			calibFiles[$numCalibs]=$entry
	        	calibs[$numCalibs]=$runnumber
			calibDrsFiles[$numCalibs]=$DrsGain
			calibDrsPedestalFiles[$numCalibs]=$DrsPedestal
			calibsRoi[$numCalibs]=`/opt/FACT++/fitsdump -h -t DrsCalibration $entry  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
			calibsRoiTM[$numCalibs]=`/opt/FACT++/fitsdump -h -t DrsCalibration $entry  2>/dev/null | grep NTM | grep -E -o '[0-9]{1,4}'`
				#echo "Calib file #"$runnumber" found. Roi=${calibsRoi[$numCalibs]} RoiTM="${calibsRoiTM[$numCalibs]}
	                numCalibs=`expr $numCalibs + 1`
		fi
	       	continue
        fi

       #if not calib, then look if this is a calibration run (i.e. run that created the drs calib)	
	correspondingDrs=$rawFolder/$filename.drs.fits.gz
	if [ -f $correspondingDrs ]
	then
		continue
	fi
       #if regular run, add its run number, and figure out a proper calibration file
        data[$numData]=$runnumber
	calibFound="false"
	partialCalibFound="false"
	calibIndex=`expr $numCalibs - 1`

	#if there is no calib available, report error
	if [ "$numCalibs" == "0" ]
	then
		calibFound="notFound"
	fi
	while [ "$calibFound" == "false" ]
	do
		roi=`/opt/FACT++/fitsdump -h -t Events $entry  2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
         	roiTM=`/opt/FACT++/fitsdump -h -t Events $entry  2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
		if [ "$roi" == "${calibsRoi[$calibIndex]}" ] && [ "$roiTM" == "${calibsRoiTM[$calibIndex]}" ]
		then
			if [ "${calibDrsFiles[$calibIndex]}" != "" ] && [ "${calibDrsPedestalFiles[$calibIndex]}" != "" ]
			then
				calibFound="true"
				break
			else
				if [ "$partialCalibFound" == "false" ]
				then
					partialCalibFound=$calibIndex
				fi
			fi
		fi
		calibIndex=`expr $calibIndex - 1`
		if [ "$calibIndex" == "-1" ]
		then
		  	calibFound="notFound"
		fi
	done
	
	if [ "$partialCalibFound" != "false" ] && [ "$calibFound" != "true" ]
	then
		calibFound=""
		calibIndex=$partialCalibFound
	fi
	
	if [ "$calibFound" == "true" ]
	then
		calibFound="complete"
	fi
	
        if [ "$calibFound" == "notFound" ]
        then
		echo "No suitable calibration file could be found for run $runnumber" | tee -a $logfile 2>&1
		calibFileString="NULL"
		calibDrsString="NULL"
		calibDrsPedestalString="NULL"
	else
		echo "Found $calibFound calibration "${calibs[$calibIndex]}" for run "$runnumber" with Roi="$roi" and roiTM="$roiTM | tee -a $logfile 2>&1
		calibFileString=${calibFiles[$calibIndex]}
		calibDrsString=${calibDrsFiles[$calibIndex]}
		calibDrsPedestalString=${calibDrsPedestalFiles[$calibIndex]}
		if [ "$calibDrsString" == "" ]
		then
			calibDrsString="NULL"
		fi
		if [ "$calibDrsPedestalString" == "" ]
		then
			calibDrsPedestalString="NULL"
		fi
	fi
	
	#File is valid. get its related informations
	currentEntry=$entry
	rm -f $keywordsFile
	rm -f $dataFile
	writeHeaderKeysToInputFile
	if [ "$doNotDoThisEntry" == "true" ]
	then
		continue
	fi
	#Header keys written for raw data. do the same for related aux data
	echo "Events BINTABLE URL ../../../"$entry" 1 1 Events" >> $dataFile
	if [ "$calibFileString" != "NULL" ]
	then
		echo "DrsCalibration BINTABLE URL ../../../"$calibFileString" 1 1 Pedestal" >> $dataFile
	fi
	if [ "$calibDrsString" != "NULL" ]
	then
		echo "DrsCalibration BINTABLE URL ../../../"$calibDrsString" 1 1 Drs_Gain" >> $dataFile
	fi
	if [ "$calibDrsPedestalString" != "NULL" ]
	then
		echo "DrsCalibration BINTABLE URL ../../../"$calibDrsPedestalString" 1 1 Drs_Pedestal" >> $dataFile
	fi
	echo "DRIVE_CONTROL_TRACKING_POSITION BINTABLE URL ../../../"$trackingFile" 1 1 Tracking_Position" >> $dataFile
	echo "FTM_CONTROL_TRIGGER_RATE BINTABLE URL ../../../"$triggerFile" 1 1 Trigger_Rate" >> $dataFile
	echo "FTM_CONTROL_STATIC_DATA BINTABLE URL ../../../"$staticFile" 1 1 Thresholds" >> $dataFile
	echo "BIAS_CONTROL_VOLTAGE BINTABLE URL ../../../"$voltagesFile" 1 1 Voltages" >> $dataFile
	echo "BIAS_CONTROL_CURRENT BINTABLE URL ../../../"$currentFile" 1 1 Currents" >> $dataFile
		
	if [ "$calibFound" != "notFound" ]
	then	
		#write info to Werner's file
		if [ "$runtype" == "custom" ]
		then
			runtype="custom____________"
		fi
		if [ "$runtype" == "data" ]
		then
			runtype="data______________"
		fi
		if [ "$runtype" == "drs-gain" ]
		then
			runtype="drs-gain__________"
		fi
		if [ "$runtype" == "drs-gain-ext" ]
		then
			runtype="drs-gain-ext______"
		fi
		if [ "$runtype" == "drs-pedestal" ]
		then
			runtype="drs-pedestal______"
		fi
		if [ "$runtype" == "drs-pedestal-ext" ]
		then
			runtype="drs-pedestal-ext__"
		fi
		if [ "$runtype" == "drs-time" ]
		then
			runtype="drs-time__________"
		fi
		if [ "$runtype" == "drs-time-delay15" ]
		then
			runtype="drs-time-delay15__"
		fi
		if [ "$runtype" == "drs-time-delay05" ]
		then
			runtype="drs-time-delay05__"
		fi
		if [ "$runtype" == "drs-time-delay20" ]
		then
			runtype="drs-time-delay20__"
		fi
		if [ "$runtype" == "drs-time-upshifted" ]
		then
			runtype="drs-time-upshifted"
		fi
		if [ "$runtype" == "light-pulser-ext" ]
		then
			runtype="light-pulser-ext__"
		fi
		if [ "$runtype" == "n/a" ]
		then
			runtype="n/a_______________"
		fi
		if [ "$runtype" == "ped-and-lp-ext" ]
		then
			runtype="ped-and-lp-ex_____"
		fi
		if [ "$runtype" == "pedestal" ]
		then
			runtype="pedestal__________"
		fi
		targetWerner=$targetFolder"/"$year"/"$month
		if [ ! -d $targetWerner ]
		then
			mkdir -p $targetWerner
		fi
		wernerFile=$targetWerner"/"$year$month"_001.txt"
		if [ ! -f $wernerFile ]
		then
			echo "creating "$wernerFile
			echo "#	DRS	DAT	DD	MM	YYYY	TYPE	" >> $wernerFile
		fi
		echo "	"${calibs[$calibIndex]}"	"$runnumber"	"$day"	"$month"	"$year"	"$runtype >> $wernerFile
	fi
	
	#now add the "other" slow control files, i.e. the ones that are not mandatory for analysis	
	auxTable="BIAS_CONTROL_NOMINAL"
	auxDesc="Bias_Control"
	writeExtraAuxFile
	auxTable="BIAS_CONTROL_STATE"
	auxDesc="Bias_State"	
	writeExtraAuxFile
	auxTable="DATA_LOGGER_FILENAME_NIGHTLY"
	auxDesc="Logger_Filename_Night"
	writeExtraAuxFile
	auxTable="DATA_LOGGER_FILENAME_RUN"
	auxDesc="Logger_Filename_Run"
	writeExtraAuxFile
	auxTable="DATA_LOGGER_NUM_SUBS"
	auxDesc="Logger_num_subs"
	writeExtraAuxFile
	auxTable="DATA_LOGGER_STATE"
	auxDesc="Logger_State"
	writeExtraAuxFile
	auxTable="DATA_LOGGER_STATS"
	auxDesc="Logger_Statistics"
	writeExtraAuxFile
	auxTable="DRIVE_CONTROL_POINTING_POSITION"
	auxDesc="Pointing_Position"
	writeExtraAuxFile
	auxTable="DRIVE_CONTROL_STATE"
	auxDesc="Drive_State"
	writeExtraAuxFile
	auxTable="DRIVE_CONTROL_STATUS"
	auxDesc="Drive_Status"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_DAC"
	auxDesc="FAD_DAC"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_DNA"
	auxDesc="FAD_DNA"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_DRS_CALIBRATION"
	auxDesc="FAD_Drs_Calibration"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_EVENTS"
	auxDesc="FAD_Events"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_FEEDBACK_DATA"
	auxDesc="FAD_Feedback_Data"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_FILE_FORMAT"
	auxDesc="FAD_File_Format"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_FIRMWARE_VERSION"
	auxDesc="FAD_Firmware_Version"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_PRESCALER"
	auxDesc="FAD_Prescaler"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_REFERENCE_CLOCK"
	auxDesc="FAD_Reference_Clock"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_REGION_OF_INTEREST"
	auxDesc="FAD_ROI"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_RUN_NUMBER"
	auxDesc="FAD_Run_Number"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_RUNS"
	auxDesc="FAD_Runs"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_START_RUN"
	auxDesc="FAD_Start_Run"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_STATE"
	auxDesc="FAD_State"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_STATISTICS1"
	auxDesc="FAD_Statistics_1"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_STATISTICS2"
	auxDesc="FAD_Statistics_2"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_STATS"
	auxDesc="FAD_Stats"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_STATUS"
	auxDesc="FAD_Status"
	writeExtraAuxFile
	auxTable="FAD_CONTROL_TEMPERATURE"
	auxDesc="FAD_Temperatures"
	writeExtraAuxFile
	auxTable="FEEDBACK_DEVIATION"
	auxDesc="Feedback_Deviation"
	writeExtraAuxFile
	auxTable="FEEDBACK_STATE"
	auxDesc="Feedback_State"
	writeExtraAuxFile
	auxTable="FSC_CONTROL_HUMIDITY"
	auxDesc="FSC_Humidity"
	writeExtraAuxFile
	auxTable="FSC_CONTROL_STATE"
	auxDesc="FSC_State"
	writeExtraAuxFile
	auxTable="FSC_CONTROL_TEMPERATURE"
	auxDesc="FSC_Temperature"
	writeExtraAuxFile
	auxTable="FTM_CONTROL_COUNTER"
	auxDesc="FTM_Counter"
	writeExtraAuxFile
	auxTable="FTM_CONTROL_DYNAMIC_DATA"
	auxDesc="FTM_Dynamic_Data"
	writeExtraAuxFile
	auxTable="FTM_CONTROL_FTU_LIST"
	auxDesc="FTM_FTU_List"
	writeExtraAuxFile
	auxTable="FTM_CONTROL_PASSPORT"
	auxDesc="FTM_Passeport"
	writeExtraAuxFile
	auxTable="FTM_CONTROL_STATE"
	auxDesc="FTM_State"
	writeExtraAuxFile
	auxTable="MAGIC_WEATHER_DATA"
	auxDesc="MAGIC_Weather_Data"
	writeExtraAuxFile
	auxTable="MAGIC_WEATHER_STATE"
	auxDesc="MAGIC_Weather_State"
	writeExtraAuxFile
	auxTable="MCP_STATE"
	auxDesc="MCP_State"
	writeExtraAuxFile
	auxTable="RATE_CONTROL_STATE"
	auxDesc="Rate_Control_State"
	writeExtraAuxFile
	#create the fits file
	targetNormal=$targetFolder"/"$year"/"$month"/"$day
	if [ ! -d $targetNormal ]
	then
		mkdir -p $targetNormal
	fi
	targetFile=$targetNormal"/"$year$month$day"_"$runnumber"_001.fits"
	if [ -f $targetFile ]
	then
		rm $targetFile
	fi
	fcreate $colDescFile $dataFile $targetFile "headfile="$keywordsFile
		echo "Created "$targetFile | tee -a $logfile 2>&1

done