Index: trunk/Mars/hawc/processing/DiskToDB/extract-aux-data2.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/extract-aux-data2.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/extract-aux-data2.sh	(revision 20091)
@@ -0,0 +1,256 @@
+#!/bin/bash
+#
+# Calculate aux data from auxfiles in directory DIR and write SQL query 
+# containing the aux information to OUTPUT
+#
+# Example:
+# 	bash extract-aux-data2.sh cred-file 1 /data/HE01/aux 20201111
+
+set -o nounset
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+
+# File to which the queries are written
+OUTPUT="insert-aux.sql"
+# The Mars build directory
+readonly MARS="/home/frankm/Dateien/Mars/build/"
+# The path to your Mars/build directory where the processing macros are stored
+# MACROS="/home/frankm/Dateien/Mars/hawc/processing"
+readonly MACROS="/home/frankm/Dateien/Mars/build/Masterarbeit/Datenbank/HE01/Mexico/Cfiles/"
+readonly PROGRAM=$0
+INSERT="INSERT INTO"
+
+usage()
+{
+	echo "usage: $PROGRAM [-hir] [-o outfile] [Credentials Telescope Dir Night]" 
+	echo "	-h 		display help"
+	echo "	-i 		query ignores already existing rows"
+	echo "	-r 		query replaces already existing rows(overwrites -i)"
+	echo "	-d 		delete all entries for a given Night and Telescope"
+	echo "	-o outfile	name of the SQL query file"
+	exit 1;
+}
+
+# Check for flags
+while getopts 'hirdo:' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		i) INSERT="INSERT IGNORE" ;;
+		r) INSERT="REPLACE INTO" ;;
+		d) DELETE="true" ;;
+		o) OUTPUT="${OPTARG}" ;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if Mars exists
+if [ ! -d "${MARS}" ]
+then
+	echo "Mars does not exists at ${MARS}. Please change in \
+		script ${PROGRAM}."
+	exit 1
+fi
+
+# Check if MACROS exists
+if [ ! -d "${MACROS}" ]
+then
+	echo "Macros directorey does not exists at ${MACROS}. Please change in \
+		script ${PROGRAM}."
+	exit 1
+fi
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]
+then
+	echo "Not enough arguments. Check -h for help!"
+	exit 1
+fi
+
+# File containing the access credentials for the database
+CREDENTIALS="${1}"
+# Specify the telescope for which to run the script.
+TEL="${2}"
+# The base directory of the aux-files
+DIR="${3}"
+# NIGHT
+NIGHT="${4}"
+
+
+
+# Open the query-file with the create table statement
+echo \
+"\
+CREATE TABLE IF NOT EXISTS AuxData
+(
+   Telescope       TINYINT   UNSIGNED  NOT NULL,
+   NIGHT           INT       UNSIGNED  NOT NULL,
+   RUNID           MEDIUMINT UNSIGNED  NOT NULL,
+   TavgDRS         FLOAT,
+   TrmsDRS         FLOAT,
+   Umed            FLOAT,
+   Uavg            FLOAT,
+   Udev            FLOAT,
+   Urms            FLOAT,
+   Imed            FLOAT,
+   Iavg            FLOAT,
+   Idev            FLOAT,
+   Irms            FLOAT,
+   Tmed            FLOAT,
+   Tavg            FLOAT,
+   Tdev            FLOAT,
+   Trms            FLOAT,
+   TavgPSU         FLOAT,
+   TotalOnTime     FLOAT,
+   TotalDeadTime   FLOAT,
+   TavgFTM         FLOAT,
+   ThresholdChMin  MEDIUMINT UNSIGNED,
+   ThresholdChMed  FLOAT,
+   ThresholdChMax  MEDIUMINT UNSIGNED,
+   ThresholdSumMin MEDIUMINT UNSIGNED,
+   ThresholdSumMax MEDIUMINT UNSIGNED,
+   RateCh0         FLOAT,
+   RateCh1         FLOAT,
+   RateCh2         FLOAT,
+   RateCh3         FLOAT,
+   RateCh4         FLOAT,
+   RateCh5         FLOAT,
+   RateCh6         FLOAT,
+   RateCh7         FLOAT,
+   RateSum0        FLOAT,
+   RateSum1        FLOAT,
+   PRIMARY KEY(Telescope, NIGHT, RUNID)
+);
+"\ > ${OUTPUT}
+
+if [ ${DELETE} = "true" ]
+then
+	echo "DELETE FROM AuxData" >> ${OUTPUT}
+	echo "WHERE Telescope = ${TEL}" >> ${OUTPUT}
+	echo "AND NIGHT = ${NIGHT};" >> ${OUTPUT}
+	echo "" >> ${OUTPUT}
+fi
+
+echo \
+"\
+${INSERT} AuxData
+SELECT * FROM
+( VALUES
+"\ >> ${OUTPUT}
+
+# Get all available runs and their start and stop time
+# note that no check is done on the stop time!
+echo \
+"\
+SELECT 
+    	NIGHT, RUNID, TSTARTI+TSTARTF, TSTOPI+TSTOPF
+FROM 
+    	RawData
+WHERE
+    	Telescope=${TEL}
+AND
+	NIGHT=${NIGHT}\
+"\
+ |  mysql \
+       --defaults-file=${CREDENTIALS} \
+       --skip-column-names \
+       --batch --raw \
+       --compress \
+ | \
+while read -r -a LINE
+do
+  # Extract night, runid, begin and end
+  NIGHT=${LINE[0]}
+  RUNID=${LINE[1]}
+  BEG=${LINE[2]}
+  END=${LINE[3]}
+
+  # Split night into year, month, day
+  # Base path for the night
+  FNAME="${DIR}"/${NIGHT:0:4}/${NIGHT:4:2}/${NIGHT:6:2}/${NIGHT}
+
+  echo ${FNAME}
+
+  cd ${MARS}
+
+  # Return corresponding data for DRS temperatures from FAD_CONTROL_TEMPEREATURE
+  DRS_TEMP=`root -b -q -l "${MACROS}"/drstemp.C\(\""${FNAME}".FAD_CONTROL_TEMPERATURE.fits\","${BEG}","${END}"\) | grep result | cut -f2- -d" "`
+  if [ ! -n "${DRS_TEMP}" ]
+  then
+  	DRS_TEMP="NULL, NULL"
+  fi
+  echo ${DRS_TEMP}
+
+  # Return corresponding data for currents from BIAS_CONTROL_DYNAMIC_DATA
+  BIAS_DATA=`root -b -q -l "${MACROS}"/currents.C\(\""${FNAME}".BIAS_CONTROL_DYNAMIC_DATA.fits\","${BEG}","${END}"\) | grep result | cut -f2- -d" "`
+  if [ ! -n "${BIAS_DATA}" ]
+  then  
+  	BIAS_DATA="NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL"
+  fi
+  echo ${BIAS_DATA}
+
+  # Return corresponding data for trigger information from FTM_CONTROL_DATA
+  FTM_DATA=`root -b -q -l "${MACROS}"/trigger.C\(\""${FNAME}".FTM_CONTROL_DATA.fits\","${BEG}","${END}"\) | grep result | cut -f2- -d" "`
+  if [ ! -n "${FTM_DATA}" ]
+  then
+  	FTM_DATA="NULL, NULL, NULL, NULL"
+  fi
+  echo ${FTM_DATA}
+
+  # Return corresponding data for threshold seeting from FTU_CONTROL_DATA
+  FTU_DATA=`root -b -q -l "${MACROS}"/threshold.C\(\""${FNAME}".FTU_CONTROL_DATA.fits\","${BEG}","${END}"\) | grep result | cut -f2- -d" "`
+  if [ ! -n "${FTU_DATA}" ]
+  then
+	FTU_DATA="NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL"
+  fi
+  echo ${FTU_DATA}
+
+  cd -
+
+  # Write the result as one row to the query file
+  echo "/* "${FNAME}" */"          	>> ${OUTPUT}
+  echo "ROW("      			>> ${OUTPUT}
+  echo " "${TEL}, ${NIGHT}, ${RUNID},	>> ${OUTPUT}
+  echo " "${DRS_TEMP},              	>> ${OUTPUT}
+  echo " "${BIAS_DATA},          	>> ${OUTPUT}
+  echo " "${FTM_DATA},              	>> ${OUTPUT}
+  echo " "${FTU_DATA}                	>> ${OUTPUT}
+  echo "),"                         	>> ${OUTPUT}
+
+done
+
+# Delete last comma and replace it with closing parintheses
+sed -i '$s/,/)/' "${OUTPUT}"
+
+# Finish the query file with defining the column names
+echo "\
+AS
+v(
+   Telescope, NIGHT, RUNID,
+   TavgDRS, TrmsDRS,
+   Umed, Uavg, Udev, Urms,
+   Imed, Iavg, Idev, Irms,
+   Tmed, Tavg, Tdev, Trms,
+   TavgPSU, TotalOnTime, TotalDeadTime, TavgFTM,
+   ThresholdChMin, ThresholdChMed, ThresholdChMax, 
+   ThresholdSumMin, ThresholdSumMax, 
+   RateCh0, RateCh1, RateCh2, RateCh3, RateCh4, RateCh5, RateCh6, RateCh7,
+   RateSum0, RateSum1
+   );
+"\ >> ${OUTPUT}
Index: trunk/Mars/hawc/processing/DiskToDB/extract-raw-header2.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/extract-raw-header2.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/extract-raw-header2.sh	(revision 20091)
@@ -0,0 +1,217 @@
+#!/bin/bash
+
+# Read fits files from DIR for a given telescope TEL and night NIGHT and write 
+# an SQL query to OUTPUT for uploading the raw header information. 
+#
+# Example:
+# 	bash extract-raw-header2.sh 1 /data/HE01/raw 20201111
+#
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+
+# define variables
+OUTPUT=insert-raw.sql
+readonly FITSDUMP="/home/frankm/Dateien/FACT++_0/build/fitsdump"
+readonly PROGRAM=$0
+INSERT="INSERT INTO"
+
+# Check if fitsdump exists
+if [ ! -f "${FITSDUMP}" ]
+then
+	echo "ERROR: fitsdump does not exists at ${FITSDUMP}. Please change in script ${PROGRAM}." >&2
+	exit 1
+fi
+
+usage()
+{
+	echo "usage: $PROGRAM [-hir] [-o outfile] [Telescope Directory Night]"
+	echo "	-h 		display help"
+	echo "	-i 		query ignores already existing rows"
+	echo "	-r 		query replaces already existing rows(overwrites -i)"
+	echo "	-d 		delete all entries for a given Night and Telescope"
+	echo "	-o outfile	name of the SQL query file"
+	exit 1;
+}
+
+# Check for flags
+while getopts 'hirdo:' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		i) INSERT="INSERT IGNORE" ;;
+		r) INSERT="REPLACE INTO" ;;
+		d) DELETE="true" ;;
+		o) OUTPUT="${OPTARG}" ;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ]
+then
+	echo "ERROR: Not enough arguments. Check -h for help!" >&2
+	exit 1
+fi
+
+TEL="${1}"
+DIR="${2}"
+NIGHT="${3}"
+
+readonly SUBDIR="${DIR}"/${NIGHT:0:4}/${NIGHT:4:2}/${NIGHT:6:2}
+
+# Keywords to grep from fitsdump
+readonly TOGREP='NIGHT\|RUNID\|TSTART\|TSTOP\|STEP\|DATE\|NROI[^T]\|DAC[0-7]\|DNA[01][^0-9]\|FWVER[01][^0-9]\|DRSCALIB\|NTRG\|NTRG\|ZNAXIS2\|REFCLK\|ZRATIO\|RUNTYPE'
+# Refference declarations for checking the header formatting
+readonly SEARCHSHORT='DAC0\|DAC1\|DAC2\|DAC3\|DAC4\|DAC5\|DAC6\|DAC7\|`DATE`\|`DATE-END`\|`DATE-OBS`\|DNA0\|DNA1\|DRSCALIB\|FWVER0\|FWVER1\|NIGHT\|NROI\|NTRG\|NTRGEXT1\|NTRGEXT2\|NTRGLPE\|NTRGLPI\|NTRGMISC\|NTRGPED\|NTRGTIM\|REFCLK\|RUNID\|RUNTYPE\|TSTARTF\|TSTARTI\|TSTOPF\|TSTOPI\|ZNAXIS2\|ZRATIO'
+readonly SEARCHLONG='DAC0\|DAC1\|DAC2\|DAC3\|DAC4\|DAC5\|DAC6\|DAC7\|`DATE`\|`DATE-END`\|`DATE-OBS`\|DNA0\|DNA1\|DRSCALIB\|DRSSTEP\|FWVER0\|FWVER1\|NIGHT\|NROI\|NTRG\|NTRGEXT1\|NTRGEXT2\|NTRGLPE\|NTRGLPI\|NTRGMISC\|NTRGPED\|NTRGTIM\|REFCLK\|RUNID\|RUNTYPE\|TSTARTF\|TSTARTI\|TSTOPF\|TSTOPI\|ZNAXIS2\|ZRATIO'
+
+readonly REFSHORT=`echo "${SEARCHSHORT}" | grep -o "${SEARCHSHORT}"`
+readonly REFLONG=`echo "${SEARCHLONG}" | grep -o "${SEARCHLONG}"`
+readonly CNTSHORT=34
+readonly CNTLONG=35
+
+
+echo \
+"\
+CREATE TABLE IF NOT EXISTS RawData
+(
+	Telescope tinyint UNSIGNED NOT NULL,
+	NIGHT int UNSIGNED NOT NULL,
+	RUNID mediumint UNSIGNED NOT NULL,
+	\`DATE-OBS\` datetime(6) NOT NULL,
+	\`DATE-END\` datetime(6) NOT NULL,
+	RUNTYPE tinyint UNSIGNED NOT NULL,
+	DRSSTEP tinyint UNSIGNED DEFAULT NULL,
+	NROI mediumint UNSIGNED NOT NULL,
+	ZNAXIS2 bigint UNSIGNED NOT NULL,
+	NTRG int UNSIGNED NOT NULL,
+	NTRGMISC int UNSIGNED NOT NULL,
+	NTRGPED int UNSIGNED NOT NULL,
+	REFCLK float NOT NULL,
+	ZRATIO float NOT NULL,
+	DAC0 mediumint UNSIGNED NOT NULL,
+	DAC1 mediumint UNSIGNED NOT NULL,
+	DAC2 mediumint UNSIGNED NOT NULL,
+	DAC3 mediumint UNSIGNED NOT NULL,
+	DAC4 mediumint UNSIGNED NOT NULL,
+	DAC5 mediumint UNSIGNED NOT NULL,
+	DAC6 mediumint UNSIGNED NOT NULL,
+	DAC7 mediumint UNSIGNED NOT NULL,
+	\`DATE\` timestamp NOT NULL,
+	DRSCALIB tinyint(1) NOT NULL,
+	NTRGEXT1 int UNSIGNED NOT NULL,
+	NTRGEXT2 int UNSIGNED NOT NULL,
+	NTRGLPE int UNSIGNED NOT NULL,
+	NTRGLPI int UNSIGNED NOT NULL,
+	NTRGTIM int UNSIGNED NOT NULL,
+	TSTARTF double NOT NULL,
+	TSTARTI mediumint UNSIGNED NOT NULL,
+	TSTOPF double NOT NULL,
+	TSTOPI mediumint UNSIGNED NOT NULL,
+	DNA0 bigint UNSIGNED NOT NULL,
+	DNA1 bigint UNSIGNED NOT NULL,
+	FWVER0 mediumint UNSIGNED NOT NULL,
+	FWVER1 mediumint UNSIGNED NOT NULL,
+	PRIMARY KEY (Telescope, NIGHT, RUNID),
+	KEY (RUNTYPE)
+); 
+"\ > ${OUTPUT}
+
+if [ ${DELETE} = "true" ]
+then
+	echo "DELETE FROM RawData" >> ${OUTPUT}
+	echo "WHERE Telescope = ${TEL}" >> ${OUTPUT}
+	echo "AND NIGHT = ${NIGHT};" >> ${OUTPUT}
+	echo "" >> ${OUTPUT}
+fi
+
+
+# Loop over all input arguments
+for ROOT in "${SUBDIR}"/* 
+do
+
+	# check if file extension is correct
+	if [ "${ROOT: -8}" == ".fits.fz" ]
+	then
+		echo "${ROOT}"
+	else
+		echo "${ROOT} has wrong file extension and will not be processed."
+		continue
+	fi
+
+	# Extract the header from the fits file with fitsdump, 
+	# the following chain properly formats the output, 
+	# replaces T/F with true/false, replaces run-ytpes by numeric 
+	# types, adds enclosures for SQL variable names where
+	# necessary, removes enclosures around hex-numbers and adds 
+	# commas after all lines except the last one
+	# FIXME: fitsdump -h alway returns error. Must be fixed in FACT++
+	#	 so pipe errors from this are ignored. Also fitsdump outputs 
+	# 	 some info to filedescriptor 2. So error outputs are now
+	# 	 redirected to filedescriptor 4, which redirects to LOG.
+	set +o pipefail
+	RESULT=`${FITSDUMP} -h "${ROOT}" 2>&4 \
+		| grep "${TOGREP}" \
+		| grep -v CHECKSUM \
+		| cut -c4-  \
+		| cut -f1 -d\/ \
+		| sed "s/'drs-pedestal'/1/g" \
+		| sed "s/'drs-gain'/2/g" \
+		| sed "s/'pedestal'/3/g" \
+		| sed "s/'data'/4/g" \
+		| sed "s/'custom'/0/g" \
+		| sed 's/\ T\ /true/g' \
+		| sed 's/\ F\ /false/g' \
+		| sed 's/^DATE-END/\`DATE-END\`/' \
+		| sed 's/^DATE-OBS/\`DATE-OBS\`/' \
+		| sed 's/^DATE[^-]/\`DATE\`/' \
+		| sed "s/^\(.*\)'\(0x[0-9a-f]*\)'\(.*\)$/\1\2\3/g" \
+		| sed '$!s/$/,/'`
+	set -o pipefail
+
+	# compare to refference header and count commas
+	CHECKSHORT=`echo ${RESULT} | grep -o "${SEARCHSHORT}"`
+	CHECKLONG=`echo ${RESULT} | grep -o "${SEARCHLONG}"`
+	CNTCHECK=`echo "${RESULT}" | grep -o ',' | wc -l`
+
+	# If long or short CHECK are matching, write result to file
+	if [ "${CHECKLONG}" == "${REFLONG}" ] && 
+		[ "${CNTCHECK}" == "${CNTLONG}" ]
+	then
+		echo "${INSERT} RawData SET" 		>> ${OUTPUT}
+		echo Telescope=${TEL},   		>> ${OUTPUT}
+		echo ${RESULT}                 		>> ${OUTPUT}
+		echo ";"                       		>> ${OUTPUT}
+	elif [ "${CHECKSHORT}" == "${REFSHORT}" ] && 
+		[ "${CNTCHECK}" == "${CNTSHORT}" ]
+	then
+		echo "${INSERT} RawData SET" 		>> ${OUTPUT}
+		echo Telescope=${TEL},   		>> ${OUTPUT}
+		echo ${RESULT}                 		>> ${OUTPUT}
+		echo ";"                       		>> ${OUTPUT}
+	else
+		echo "${ROOT} header not valid!"
+	fi
+
+done
Index: trunk/Mars/hawc/processing/DiskToDB/run-callisto2.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/run-callisto2.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/run-callisto2.sh	(revision 20091)
@@ -0,0 +1,195 @@
+#!/bin/bash
+#
+# Runs callisto macro on all files in directory DIR for a given telescope TEL
+# and night NIGHT and writes output to DIR/../callisto
+#
+# Example:
+# 	bash run-callisto2.sh cred-file 1 /data/HE01/raw 20201111
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+readonly CALLISTO=callisto
+# The .success file in /data/star will be resetted
+readonly STAR=star
+# Directory to Mars environment (usually Mars/build)
+readonly MARS="/home/frankm/Dateien/Mars/build/"
+# Absolut path to macros
+readonly MACROS="/home/frankm/Dateien/Mars/hawc"
+readonly PROGRAM=$0
+REPLACE=false
+
+usage()
+{
+	echo "usage: $PROGRAM [-hr] [Credentials Telescope Dir Night]" 
+	echo "	-h 		display help"
+	echo "	-r 		replace data"
+	exit 1;
+}
+
+# Check for flags
+while getopts 'hr' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		r) REPLACE=true ;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if Mars exists
+if [ ! -d "${MARS}" ]
+then
+	echo "Mars does not exists at ${MARS}. Please change in \
+		script ${PROGRAM}."
+	exit 1
+fi
+
+# Check if MACROS exists
+if [ ! -d "${MACROS}" ]
+then
+	echo "Macros directorey does not exists at ${MACROS}. Please change in \
+		script ${PROGRAM}."
+	exit 1
+fi
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]
+then
+	echo "Not enough arguments. Check -h for help!"
+	exit 1
+fi
+
+# File containing the access credentials for the database
+CRED="${1}"
+# Specify the telescope for which to run the script.
+TEL="${2}"
+# The data is expected to be found in /data/raw and data/callisto
+DATA="${3}"
+NIGHT="${4}"
+
+# Get all runs that can (and should) be calibrated
+# Pipe the output to mysql and read the
+# mysql output line-by-libe
+echo \
+"\
+SELECT 
+    	NIGHT, RUNID, DrsNight, DrsRunID
+FROM 
+	Calibration
+WHERE
+    	Telescope=${TEL}
+AND	
+	NIGHT=${NIGHT}
+ORDER BY
+    	NIGHT, RUNID\
+"\
+|  mysql \
+       --defaults-file=${CRED} \
+       --skip-column-names \
+       --batch --raw \
+       --compress \
+       | \
+       while read -r -a LINE
+       do
+	       # Extract night/runid for data file and calibration file
+	       DATNIGHT=${LINE[0]}
+	       DATRUNID=${LINE[1]}
+	       DRSNIGHT=${LINE[2]}
+	       DRSRUNID=${LINE[3]}
+
+	       # Formatting of the file paths and names
+	       DATPATH=${DATNIGHT:0:4}/${DATNIGHT:4:2}/${DATNIGHT:6:2}
+	       DRSPATH=${DRSNIGHT:0:4}/${DRSNIGHT:4:2}/${DRSNIGHT:6:2}
+
+	       PREFIX=`printf ${DATNIGHT}_%03d ${DATRUNID}`
+
+	       DATNAME=`printf ${DATNIGHT}_%03d.fits.fz ${DATRUNID}`
+	       DRSNAME=`printf ${DRSNIGHT}_%03d.drs.fits ${DRSRUNID}`
+
+	       OUT="${DATA}"/../${CALLISTO}/${DATPATH}
+
+	       echo DAT=${DATNAME} [${DATPATH}]
+	       echo DRS=${DRSNAME} [${DRSPATH}]
+
+
+	       if [ ${REPLACE} = true ]
+	       then
+		       echo "deleting ${OUT}/.${PREFIX}.succsess !!!"
+		       rm -f "${OUT}"/.${PREFIX}.success
+	       fi
+
+	       # Check if not yet successfully processed
+	       # Removing this file can be used to trigger a re-processing
+	       # the next time this script is executed
+	       if [ ! -f "${OUT}/.${PREFIX}.success" ]
+	       then
+
+		       cd ${MARS}
+
+		       mkdir -p "${OUT}"
+
+		       # Trigger reprocessing of the process-fils in the star directory
+		       rm -f "${DATA}/${STAR}/${DATPATH}/.${PREFIX}.*"
+
+		       # Flag that a process is running
+		       rm -f "${OUT}"/.${PREFIX}.done
+		       rm -f "${OUT}"/.${PREFIX}.success
+
+		       touch "${OUT}"/.${PREFIX}.running
+
+		       echo ${DATA}
+
+		       # Execute the calibration and write output to log-file
+		       root -b -q -l ${MACROS}/callisto.C'("'"${DATA}"/${DATPATH}/${DATNAME}'","'"${DATA}"/${DRSPATH}/${DRSNAME}'","'"${OUT}"'")' \
+			       2>&1 | tee "${OUT}"/${PREFIX}.log
+
+		       RC=${PIPESTATUS[0]}
+
+		       CALLERR=`grep "Error" "${OUT}"/${PREFIX}.log || true`
+		       if [ -n "${CALLERR}" ]
+		       then
+			       echo "ERROR: Line $LINENO error in callisto.C" >&2
+			       echo "error was: ${CALLERR}" >&2
+			       exit 1
+		       fi
+
+		       # Remember exit status of callisto
+		       echo RC=${RC} | tee -a "${OUT}"/${PREFIX}.log
+		       echo ${RC} > "${OUT}"/.${PREFIX}.done
+		       # Processing is finished
+		       rm -f "${OUT}"/.${PREFIX}.running
+
+		       # If processing was successfull write corresponding flag
+		       if [ "${RC}" == "0" ]
+		       then
+			       touch "${OUT}"/.${PREFIX}.success
+		       fi
+
+		       cd -
+
+		       echo "-----End of first iteration-----"
+	       else
+		       echo Skipped.
+	       fi
+
+       done
Index: trunk/Mars/hawc/processing/DiskToDB/run-root2sql2.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/run-root2sql2.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/run-root2sql2.sh	(revision 20091)
@@ -0,0 +1,145 @@
+#/bin/bash
+#
+# Uploads star or callisto data to SQL database
+#
+# Example:
+# 	bash run-root2sql2.sh -I cred-file starrc 1 /data/HE01/star 20201111
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+# Path to the executable
+readonly ROOT2SQL="/home/frankm/Dateien/FACT++_0/build/root2sql"
+readonly PROGRAM=${0}
+
+# Check if root2sql  exists
+if [ ! -f "${ROOT2SQL}" ]
+then
+	echo "root2sql does not exists at ${ROOT2SQL}. Please change in script ${PROGRAM}."
+	exit 1
+fi
+
+usage()
+{
+	echo "usage: $PROGRAM [-hIY] [Credentials Resource Telescope Dir Night]"
+	echo "	-h 		display help"
+	echo "	-I 		is Star data"
+	echo "	-Y 		is Callisto data"
+	exit 1;
+}
+
+# Check for flags
+while getopts 'hIY' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		I) SUFFIX=I;;
+		Y) SUFFIX=Y;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ] || [ -z "${5}" ]
+then
+	echo "Not enough arguments. Check -h for help!"
+	exit 1
+fi
+
+# File containing the access credentials for the database
+CRED="${1}"
+# Resource file to use (should match the input files)
+RESOURCE="${2}"
+# Specify the telescope for which to run the script.
+TEL="${3}"
+# Path where the data is stored... can be callisto or star
+BASEIN="${4}"
+# Night
+NIGHT="${5}"
+
+# The table to insert the data into (and create if not exists)
+if [ $SUFFIX = I ]
+then
+	TABLE=Star
+elif [ $SUFFIX = Y ]
+then
+	TABLE=Callisto
+else
+	echo "Wrong suffix."
+	exit 1
+fi
+
+# Get all runs that can (and should) be calibrated
+# Pipe the output to mysql and read the
+# mysql output line-by-libe
+echo \
+"\
+SELECT 
+	NIGHT, RUNID
+FROM 
+	Calibration
+WHERE
+	Telescope = ${TEL}
+AND
+	NIGHT = ${NIGHT}
+ORDER BY
+	NIGHT, RUNID\
+
+"\
+ |  mysql \
+       --defaults-file=${CRED} \
+       --skip-column-names \
+       --batch --raw \
+       --compress \
+ | \
+while read -r -a LINE
+do
+	# Get NIGHT and RUNID of all files
+	NIGHT=${LINE[0]}
+	RUNID=${LINE[1]}
+
+  	# Format file and path names
+	DIR="${BASEIN}"/${NIGHT:0:4}/${NIGHT:4:2}/${NIGHT:6:2}
+
+	PREFIX=`printf ${NIGHT}_%03d ${RUNID}`
+
+	echo ""
+	echo "${DIR}"/${PREFIX}
+
+	# Check if processing was successfull
+	if [ -f "${DIR}"/.${PREFIX}.success ]
+	then
+
+		${ROOT2SQL} "${DIR}"/${PREFIX}_${SUFFIX}.root \
+			-C ${RESOURCE} \
+			--table=${TABLE} \
+			--const.Telescope=${TEL} \
+			--const.NIGHT=${NIGHT} \
+			--const.RUNID=${RUNID} \
+			2>&1 | tee "${DIR}"/${PREFIX}-root2sql.log
+
+		echo RC=${PIPESTATUS[0]} >> "${DIR}"/${PREFIX}-root2sql.log
+	else
+		echo Skipped.
+	fi
+
+done
Index: trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh	(revision 20091)
@@ -0,0 +1,293 @@
+#!/bin/bash
+#
+# Synchronizes the HAWC's Eye data on disk with the SQL database.
+#
+# The following scripts must be in the same directory as this script and 
+# the named variables must be eddited in those files.
+#	run-scripts.sh (this script)
+# 		TEL
+# 		DIR
+# 		CRED
+# 		CALLISTORC
+# 		STARRC
+# 	update-file-table2.sh
+# 	extract-raw-header2.sh
+#		FITSDUMP
+# 	write-calibration-query.sh
+# 	extract-aux-data2.sh
+#		MARS
+#		MACROS
+# 	run-callisto2.sh
+#		MARS
+#		MACROS
+# 	run-star2.sh
+#		MARS
+#		MACROS
+# 	run-root2sql2.sh
+#		ROOT2SQL
+# 
+# Example:
+#	bash run-scripts.sh
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+
+# Filename
+readonly SCRPTN=`basename $0 .sh`
+
+# For lockfile
+readonly LOCK="/tmp/${SCRPTN}.lock"
+
+function LockExit()
+{
+	echo "ERROR: One instance of ${SCRPTN} already running" >&2
+	exit 1
+}
+
+function DescExit()
+{
+	echo "ERROR: Filedescriptor 200 already taken" >&2
+	exit 1
+}
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+exec 200>${LOCK} || DescExit
+flock -n 200 || LockExit
+
+# For logfile
+mkdir -p log
+readonly DATE=`date +%Y%m%d_%H%M`
+readonly LOG="./log/${SCRPTN}_${DATE}.log"
+# exec 3>&1 1>>${LOG}
+# exec 4>>${LOG}
+exec 3>&1
+exec 4>&1
+
+echo "Start processing data"
+
+function DeleteEntries()
+{
+	# Delete entries form table ${3} for a given Telescope and Night
+	# Arguments: Credentials Telescope Table Night
+	mysql --defaults-file="${1}" \
+		--skip-column-names \
+		--batch \
+		--raw \
+		--compress \
+		-e "DELETE FROM ${3} WHERE Telescope = ${2} AND NIGHT = ${4};"
+}
+
+function SetStatus()
+{
+	# Query with process update
+	# Arguments: Credentials Telescope Status Night
+	mysql --defaults-file="${1}" \
+		--skip-column-names \
+		--batch \
+		--raw \
+		--compress \
+		-e "UPDATE DataOnDisk SET STATUS = ${3} WHERE NIGHT = ${4} \
+		AND Telescope = ${2};"
+}
+
+function GetStatus()
+{
+	# Return status of Night from database
+	# Arguments: Credentials Telescope Night
+	local STATUS=`mysql --defaults-file="${1}" \
+		--skip-column-names \
+		--batch \
+		--raw \
+		--compress \
+		-e "SELECT DISTINCT STATUS FROM DataOnDisk WHERE NIGHT = ${3} \
+		AND Telescope = ${2};"`
+
+	if [ -z ${STATUS} ]
+	then
+		echo "ERROR: Status does not exist for telescope: ${2} and night: ${3}" >&2
+		exit 1
+	fi
+
+	if [ `echo "${STATUS}" | wc -l` -ne "1" ]
+	then
+		echo "ERROR: Not all status values are the same for the night: ${3}!" >&2
+		exit 1
+	fi
+
+	echo "${STATUS}"
+}
+
+# Define variables
+# Telescope for which to process
+readonly TEL=1
+# Directory in which the raw data is located
+readonly DIR='/run/media/frankm/Neuer Datenträger/data/DATA/HE0'${TEL}
+# Credentials for the SQL database
+readonly CRED='../credentials-read-only.cnf'
+# Credentials for root2sql for callisto
+readonly CALLISTORC='../root2sql-callisto.rc'
+# Credentials for root2sql for star
+readonly STARRC='../root2sql-star.rc'
+# SQL outputs
+mkdir -p ./queries
+readonly ONDISK='./queries/insert-OnDisk.sql'
+readonly RAW='./queries/insert-raw.sql'
+readonly CALIB='./queries/find-calibration2.sql'
+readonly AUX='./queries/insert-aux.sql'
+
+# Create Database of existing files
+# Create SQL query with:
+# (Telescope, Night, RUNID, ISDRSFILE, ISPROCESSED).
+echo "Updating file table"
+bash ./update-file-table2.sh -i -o "${ONDISK}" "${TEL}" "${DIR}/raw"
+# Update DataOnDisk tabel in database
+echo "Uploading updated table"
+mysql --defaults-file="${CRED}" < "${ONDISK}"
+
+# get list of NIGHTS with unprocessed files
+echo "Getting nights to be processed"
+NIGHTS=`mysql --defaults-file="${CRED}" \
+	--skip-column-names \
+	--batch \
+	--raw \
+	--compress \
+	-e "SELECT DISTINCT NIGHT FROM DataOnDisk WHERE STATUS != 5 \
+	AND Telescope = ${TEL};"`
+
+# Check if NIGHTS is empty and dont do anythin if so.
+if [ -z "${NIGHTS}" ]
+then
+	echo "Database is up to date. Nothing to do here."
+	exit 0
+fi
+
+echo "Updating database" >&3
+
+echo "Nights to process:"
+echo "${NIGHTS}"
+echo " "
+
+
+# Execute scripts on all RunId for given Telescope and Night that has
+# Loop over nights and upload to database for every loop
+for NIGHT in ${NIGHTS}
+do
+	STATUS=`GetStatus ${CRED} ${TEL} ${NIGHT}`
+	echo "Status = ${STATUS} for night = ${NIGHT}"
+	echo " "
+
+	if [ ${STATUS} -lt "1" ]
+	then
+		# Get raw data headers
+		echo "== In Status 1 =="
+		bash ./extract-raw-header2.sh -d -o "${RAW}" \
+			${TEL} \
+			"${DIR}"/raw \
+			${NIGHT}
+		echo "Uploading raw headers for night: ${NIGHT}"
+		mysql --defaults-file="${CRED}" < "${RAW}"
+		SetStatus ${CRED} ${TEL} "1" ${NIGHT}
+		echo "Finished uploading raw headers for night: ${NIGHT}"
+		echo " "
+	fi
+
+	if [ ${STATUS} -lt "2" ]
+	then
+		# Get calibration files
+		echo "== In Status 2 =="
+		bash ./write-calibration-query.sh -d -o "${CALIB}" \
+			${TEL} \
+			${NIGHT}
+		echo "Uploading calibration for night: ${NIGHT}"
+		mysql --defaults-file="${CRED}" < "${CALIB}"
+		SetStatus ${CRED} ${TEL} "2" ${NIGHT}
+		echo "Finished uploading calibration for night: ${NIGHT}"
+		echo " "
+	fi
+
+	# if [ ${STATUS} -lt "3" ]
+	if [ "a" = "b" ]
+	then
+		# Get aux data
+		echo "== In Status 3 =="
+		bash ./extract-aux-data2.sh -d -o "${AUX}" \
+			${CRED} \
+			${TEL} \
+			"${DIR}"/auxil \
+			${NIGHT}
+		echo "Uploading aux data for night: ${NIGHT}"
+		mysql --defaults-file="${CRED}" < "${AUX}"
+		SetStatus ${CRED} ${TEL} "3" ${NIGHT}
+		echo "Finished uploading aux data for night: ${NIGHT}"
+		echo " "
+	fi
+
+	if [ ${STATUS} -lt "4" ]
+	then
+		# Get callisto data
+		echo "== In Status 4 =="
+		bash ./run-callisto2.sh -r \
+			"${CRED}" \
+			${TEL} \
+			"${DIR}"/raw \
+			${NIGHT}
+
+		echo "Deleting callisto entries for night: ${NIGHT}"
+		DeleteEntries ${CRED} ${TEL} "Callisto" ${NIGHT}
+
+		echo "Uploading callisto entries for night: ${NIGHT}"
+		bash ./run-root2sql2.sh -Y \
+			"${CRED}" \
+			"${CALLISTORC}" \
+			${TEL} \
+			"${DIR}"/callisto \
+			${NIGHT}
+		SetStatus ${CRED} ${TEL} "4" ${NIGHT}
+		echo "Finished uploading callisto entries for night: ${NIGHT}"
+		echo " "
+	fi
+
+	SetStatus ${CRED} ${TEL} "4" ${NIGHT}
+	if [ ${STATUS} -lt "5" ]
+	then
+		# Get star data
+		echo "== In Status 5 =="
+		bash ./run-star2.sh -r "${CRED}" ${TEL} "${DIR}"/raw ${NIGHT}
+
+		echo "Deleting star entries for night: ${NIGHT}"
+		DeleteEntries ${CRED} ${TEL} "Star" ${NIGHT}
+
+		echo "Uploading star entries for night: ${NIGHT}"
+		bash ./run-root2sql2.sh -I \
+			"${CRED}" \
+			"${STARRC}" \
+			${TEL} \
+			"${DIR}"/star \
+			${NIGHT}
+		SetStatus ${CRED} ${TEL} "5" ${NIGHT}
+		echo "Finished uploading star entries for night: ${NIGHT}"
+		echo " "
+	fi
+
+done
+
+echo " "
+echo "Finished processing data"
Index: trunk/Mars/hawc/processing/DiskToDB/run-star2.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/run-star2.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/run-star2.sh	(revision 20091)
@@ -0,0 +1,186 @@
+#!/bin/bash
+#
+# Run star macro on directory DIR for a given telescope TEL and night NIGHT 
+# and write output to DIR/../star
+#
+# Example:
+# 	bash run-star2.sh cred-file 1 /data/HE01/raw 20201111
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+readonly CALLISTO=callisto
+# The .success file in /data/star will be resetted
+readonly STAR=star
+# Directory to Mars environment (usually Mars/build)
+readonly MARS="/home/frankm/Dateien/Mars/build/"
+# Absolut path to macros
+readonly MACROS="/home/frankm/Dateien/Mars/hawc"
+readonly PROGRAM=$0
+REPLACE=false
+
+usage()
+{
+	echo "usage: $PROGRAM [-hr] [Credentials Telescope Dir Night]" 
+	echo "	-h 		display help"
+	echo "	-r 		replace data"
+	exit 1;
+}
+
+# Check for flags
+while getopts 'hr' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		r) REPLACE=true ;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if Mars exists
+if [ ! -d "${MARS}" ]
+then
+	echo "Mars does not exists at ${MARS}. Please change in \
+		script ${PROGRAM}."
+	exit 1
+fi
+
+# Check if MACROS exists
+if [ ! -d "${MACROS}" ]
+then
+	echo "Macros directorey does not exists at ${MACROS}. Please change in \
+		script ${PROGRAM}."
+	exit 1
+fi
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]
+then
+	echo "Not enough arguments. Check -h for help!"
+	exit 1
+fi
+
+# File containing the access credentials for the database
+CRED="${1}"
+# Specify the telescope for which to run the script.
+TEL="${2}"
+# The data is expected to be found in /data/raw and data/callisto
+DATA="${3}"
+NIGHT="${4}"
+
+# Get all runs that can (and should) be calibrated
+# Pipe the output to mysql and read the
+# mysql output line-by-libe
+echo \
+"\
+SELECT 
+    	NIGHT, RUNID
+FROM 
+    	Calibration
+WHERE
+	Telescope=${TEL}
+AND
+	NIGHT=${NIGHT}
+ORDER BY
+    	NIGHT, RUNID\
+"\
+ |  mysql \
+       --defaults-file=${CRED} \
+       --skip-column-names \
+       --batch --raw \
+       --compress \
+       | \
+       while read -r -a LINE
+       do
+	       # Get NIGHT and RUNID of all files
+	       NIGHT=${LINE[0]}
+	       RUNID=${LINE[1]}
+
+  		# Formatting of the file paths and names
+		DATPATH=${NIGHT:0:4}/${NIGHT:4:2}/${NIGHT:6:2}
+
+		PREFIX=`printf ${NIGHT}_%03d ${RUNID}`
+
+		DATNAME=`printf ${PREFIX}_Y.root ${RUNID}`
+
+		OUT="${DATA}"/../${STAR}/${DATPATH}
+		IN="${DATA}"/../${CALLISTO}/${DATPATH}
+
+		echo ${DATNAME}
+		echo ${IN} " => " ${OUT}
+
+		if [ ${REPLACE} = true ]
+		then
+			echo "deleting ${OUT}/.${PREFIX}.succsess !!!"
+			rm -f "${OUT}"/.${PREFIX}.success
+		fi
+
+  		# Check if not yet successfully processed but sucessfully calibrated
+		if [[ -f "${IN}/.${PREFIX}.success" && ! -f "${IN}/.${PREFIX}.running" && ! -f "${OUT}/.${PREFIX}.success" ]]
+		then
+
+			cd ${MARS}
+
+			mkdir -p "${OUT}"
+
+     			# Flag that a process is running
+			rm -f "${OUT}"/.${PREFIX}.done
+			rm -f "${OUT}"/.${PREFIX}.success
+
+			touch "${OUT}"/.${PREFIX}.running
+
+     			# Execute image parameter calculation and write log-file
+			root -b -q -l ${MACROS}/star.C'("'"${IN}"/${DATNAME}'","'"${OUT}"'")' \
+				2>&1 | tee "${OUT}"/${PREFIX}.log
+
+     			# Remember exit status of callisto
+			RC=${PIPESTATUS[0]}
+
+			STARERR=`grep "Error" "${OUT}"/${PREFIX}.log || true`
+			BADSTRG=`grep "Error in <ReplaceSubs>: bad string number:"\
+				"${OUT}"/${PREFIX}.log || true`
+			if [ -n "${STARERR}" ] && [ -z "${BADSTRG}" ]
+			then
+				echo "ERROR: Line $LINENO error in star.C" >&2
+				echo "error was: ${STARERR}" >&2
+				exit 1
+			fi
+
+			echo RC=${RC} | tee -a "${OUT}"/${PREFIX}.log
+			echo ${RC} > "${OUT}"/.${PREFIX}.done
+
+     			# Flag that processing is finished
+			rm -f "${OUT}"/.${PREFIX}.running
+
+     			# If processing was successfull write coresponding flag
+			if [ "${RC}" == "0" ]
+			then
+				touch "${OUT}"/.${PREFIX}.success
+			fi
+
+			cd -
+
+		else
+			echo Skipped.
+		fi
+
+	done
Index: trunk/Mars/hawc/processing/DiskToDB/update-file-table2.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/update-file-table2.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/update-file-table2.sh	(revision 20091)
@@ -0,0 +1,170 @@
+#!/bin/bash
+
+# Recursively finds all .fits.fz and .drs.fits files in DIR and extract 
+# NIGHT, RUNID and if the file is a DRS file and writes an SQL query with the 
+# columns: (Telescope, NIGHT, RUNID, ISDRSFILE, ISPRECESSED) to OUTPUT.
+#
+# Example:
+# 	bash update-file-table2.sh 1 /data/HE01/raw
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+# define variables
+OUTPUT="insert-OnDisk.sql"
+INSERT="INSERT INTO"
+readonly PROGRAM=$0
+
+usage()
+{
+	echo "usage: $PROGRAM [-hir] [-o outfile] [Telescope Dir]"
+	echo "	-h 		display help"
+	echo "	-i 		query ignores already existing rows"
+	echo "	-r 		query replaces already existing rows(overwrites -i)"
+	echo "	-o outfile	name of the SQL query file"
+	exit 1;
+}
+
+# check for flags
+while getopts 'hiro:' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		i) INSERT="INSERT IGNORE" ;;
+		r) INSERT="REPLACE INTO" ;;
+		o) OUTPUT="${OPTARG}" ;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ]
+then
+	echo "ERROR: Not enough arguments. Check -h for help!" >&2
+	exit 1
+fi
+
+TEL="${1}"
+DIR="${2}"
+
+
+# Initialize the Table
+echo \
+"\
+CREATE TABLE IF NOT EXISTS DataOnDisk
+(
+	Telescope tinyint UNSIGNED NOT NULL,
+	NIGHT int UNSIGNED NOT NULL,
+	RUNID mediumint UNSIGNED NOT NULL,
+	ISDRSFILE tinyint(1) NOT NULL,
+	STATUS tinyint UNSIGNED NOT NULL,
+	Updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 
+		ON UPDATE CURRENT_TIMESTAMP,
+	PRIMARY KEY (Telescope, NIGHT, RUNID, ISDRSFILE)
+); 
+
+CREATE TEMPORARY TABLE DataOnDisk_tmp
+(
+	Telescope tinyint UNSIGNED NOT NULL,
+	NIGHT int UNSIGNED NOT NULL,
+	RUNID mediumint UNSIGNED NOT NULL,
+	ISDRSFILE tinyint(1) NOT NULL,
+	STATUS tinyint UNSIGNED NOT NULL,
+	Updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 
+		ON UPDATE CURRENT_TIMESTAMP,
+	PRIMARY KEY (Telescope, NIGHT, RUNID, ISDRSFILE)
+); 
+
+INSERT INTO DataOnDisk_tmp (Telescope, NIGHT, RUNID, ISDRSFILE, STATUS)
+VALUES
+"\ > ${OUTPUT}
+
+# Find query for .fits.fz files
+# Crop filenames from the form /nnnnnnnn_ddd.fits.fz$ to nnnnnnnn_ddd
+NRID=`find "${DIR}" -type f \
+	| grep -o "/[0-9]\{8\}_[0-9]\{3\}.fits.fz$" \
+	| cut -f2 -d "/" \
+	| cut -f1 -d "."`
+
+# Format to (Telescope, NIGHT, RUNID, ISDRSFILE, ISPROCESSED),
+FZQUERY=`paste <(for i in ${NRID}; do echo ${TEL}; done) \
+	<(echo "${NRID}" | cut -f1 -d "_") \
+	<(echo "${NRID}" | cut -f2 -d "_") \
+	<(for i in ${NRID}; do echo "false"; done) \
+	<(for i in ${NRID}; do echo "0"; done) \
+	--delimiters "," \
+	| sed 's/^/(/' \
+	| sed 's/$/),/'`
+
+# Find query for .drs.fits files
+# Crop filenames from the form nnnnnnnn_ddd.drs.fits to nnnnnnnn_ddd
+DNRID=`find "${DIR}" -type f \
+	| grep -o "/[0-9]\{8\}_[0-9]\{3\}.drs.fits$" \
+	| cut -f2 -d "/" \
+	| cut -f1 -d "."`
+
+# Format to (Telescope, NIGHT, RUNID, ISDRSFILE, ISPROCESSED),
+DRSQUERY=`paste <(for i in ${DNRID}; do echo ${TEL}; done) \
+	<(echo "${DNRID}" | cut -f1 -d "_") \
+	<(echo "${DNRID}" | cut -f2 -d "_") \
+	<(for i in ${DNRID}; do echo "true"; done) \
+	<(for i in ${DNRID}; do echo "0"; done) \
+	--delimiters "," \
+	| sed 's/^/(/' \
+	| sed 's/$/),/'`
+
+# Write query to OUTPUT file
+echo "${FZQUERY}" >> ${OUTPUT}
+echo "${DRSQUERY}" >> ${OUTPUT}
+
+# Change last comma to semicolon
+sed -i '$s/,$/;/' "${OUTPUT}"
+
+# 1) If there is data in the database that is not on disk then set status = 0
+#    for that night
+# 2) Delete all entries from the database that are not on disk
+# 3) ${INSERT} all entries into the database that are on disk
+echo \
+"\
+CREATE TEMPORARY TABLE TableNotDisk
+(
+	Telescope tinyint UNSIGNED NOT NULL,
+	NIGHT int UNSIGNED NOT NULL,
+	RUNID mediumint UNSIGNED NOT NULL,
+	ISDRSFILE tinyint(1) NOT NULL,
+	STATUS tinyint UNSIGNED NOT NULL,
+	Updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 
+		ON UPDATE CURRENT_TIMESTAMP,
+	PRIMARY KEY (Telescope, NIGHT, RUNID, ISDRSFILE)
+) SELECT * FROM DataOnDisk
+	WHERE (Telescope, NIGHT, RUNID, ISDRSFILE) NOT IN 
+	(SELECT Telescope, NIGHT, RUNID, ISDRSFILE FROM DataOnDisk_tmp);
+
+UPDATE DataOnDisk SET STATUS = 0 WHERE NIGHT IN 
+	(SELECT NIGHT FROM TableNotDisk);
+
+DELETE FROM DataOnDisk 
+	WHERE (Telescope, NIGHT, RUNID, ISDRSFILE) NOT IN 
+	(SELECT Telescope, NIGHT, RUNID, ISDRSFILE FROM DataOnDisk_tmp);
+
+${INSERT} DataOnDisk (SELECT * FROM DataOnDisk_tmp);
+"\ >> ${OUTPUT}
Index: trunk/Mars/hawc/processing/DiskToDB/write-calibration-query.sh
===================================================================
--- trunk/Mars/hawc/processing/DiskToDB/write-calibration-query.sh	(revision 20091)
+++ trunk/Mars/hawc/processing/DiskToDB/write-calibration-query.sh	(revision 20091)
@@ -0,0 +1,172 @@
+#!/bin/bash
+#
+# Write SQL query for calibration for a given telescope TEL and night NIGHT to
+# OUTPUT
+#
+# Example:
+#	bash write-calibration-query.sh 1 20201111
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+function ErrExit()
+{
+	echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
+	exit 1
+}
+
+function StrgCExit()
+{
+	echo " "
+	echo "$0 was forcefully terminated" >&2
+	exit 1
+}
+
+trap ErrExit ERR
+trap StrgCExit INT
+
+# define variables
+OUTPUT=find-calibration-file2.sql
+readonly PROGRAM=$0
+INSERT="INSERT INTO"
+
+usage()
+{
+	echo "usage: $PROGRAM [-hir] [-o outfile] [Telescope Night]"
+	echo "	-h 		display help"
+	echo "	-i 		query ignores already existing rows"
+	echo "	-r 		query replaces already existing rows(overwrites -i)"
+	echo "	-d 		delete all entries for a given Night and Telescope"
+	echo "	-o outfile	name of the SQL query file"
+	exit 1;
+}
+
+# Check for flags
+while getopts 'hirdo:' flag
+do
+	case "${flag}" in
+		h) usage ;;
+		i) INSERT="INSERT IGNORE" ;;
+		r) INSERT="REPLACE INTO" ;;
+		d) DELETE="true" ;;
+		o) OUTPUT="${OPTARG}" ;;
+		*) usage ;;
+	esac
+done
+shift $(($OPTIND-1))
+
+# Check if at least two arguments are provided
+if [ -z "${1}" ] || [ -z "${2}" ]
+then
+	echo "Not enough arguments. Check -h for help!"
+	exit 1
+fi
+
+TEL="${1}"
+NIGHT="${2}"
+
+
+echo \
+"\
+-- You might want to change the table name
+CREATE TABLE IF NOT EXISTS Calibration
+(
+	Telescope tinyint UNSIGNED NOT NULL,
+	NIGHT int UNSIGNED NOT NULL,
+	RUNID mediumint UNSIGNED NOT NULL,
+	DrsNight double NOT NULL,
+	DrsRunId double NOT NULL,
+	DeltaT double UNSIGNED NOT NULL,
+	PRIMARY KEY (Telescope, NIGHT, RUNID)
+);
+"\ > ${OUTPUT}
+
+if [ ${DELETE} = "true" ]
+then
+	echo "DELETE FROM Calibration" >> ${OUTPUT}
+	echo "WHERE Telescope = ${TEL}" >> ${OUTPUT}
+	echo "AND NIGHT = ${NIGHT};" >> ${OUTPUT}
+	echo "" >> ${OUTPUT}
+fi
+
+echo \
+"\
+CREATE TEMPORARY TABLE Calibration_tmp
+(
+   WITH Table1 AS
+   (
+      SELECT
+         -- NIGHT and RUNID of the data file
+         R1.Telescope AS Telescope,
+	 R1.NIGHT     AS NIGHT,
+         R1.RUNID     AS RUNID,
+         -- NIGHT and RUNID of the DRS file
+         R2.NIGHT     AS DrsNight,
+         R2.RUNID     AS DrsRunID,
+         -- Time difference between the two
+         LEAST(
+             ABS(UNIX_TIMESTAMP(R2.\`DATE-OBS\`)-UNIX_TIMESTAMP(R1.\`DATE-OBS\`)),
+             ABS(UNIX_TIMESTAMP(R2.\`DATE-OBS\`)-UNIX_TIMESTAMP(R1.\`DATE-END\`)),
+             ABS(UNIX_TIMESTAMP(R2.\`DATE-END\`)-UNIX_TIMESTAMP(R1.\`DATE-OBS\`)),
+             ABS(UNIX_TIMESTAMP(R2.\`DATE-END\`)-UNIX_TIMESTAMP(R1.\`DATE-END\`))
+         ) AS DeltaT,
+         -- Assign a row number to each combination of data-file and drs-file
+         -- after sorting them by time difference
+         ROW_NUMBER() OVER(PARTITION BY R1.NIGHT, R1.RUNID, R1.Telescope ORDER BY
+            LEAST(
+            ABS(UNIX_TIMESTAMP(R2.\`DATE-OBS\`)-UNIX_TIMESTAMP(R1.\`DATE-OBS\`)),
+            ABS(UNIX_TIMESTAMP(R2.\`DATE-OBS\`)-UNIX_TIMESTAMP(R1.\`DATE-END\`)),
+            ABS(UNIX_TIMESTAMP(R2.\`DATE-END\`)-UNIX_TIMESTAMP(R1.\`DATE-OBS\`)),
+            ABS(UNIX_TIMESTAMP(R2.\`DATE-END\`)-UNIX_TIMESTAMP(R1.\`DATE-END\`))
+            )
+         ) AS RowNumber
+      FROM
+         RawData R1
+      -- Join the table with itself to get all possible combinations of runs
+      CROSS JOIN
+         RawData R2
+      -- Join the DRS files with the table that tells you whether they can be used
+      -- if the table does not (yet) exists, this can be omitted
+      -- LEFT JOIN
+      --    DoNotUse ON (R2.NIGHT=DoNotUse.NIGHT AND R2.RUNID=DoNotUse.RUNID)
+      WHERE
+         -- Of course claibration must be from the same telescope
+         R1.Telescope=R2.Telescope
+      AND
+         -- Select all combinations that have a data file in R1 and a drs file (at step 1) in R2
+         ISNULL(R1.DRSSTEP) AND R2.DRSSTEP=1
+      AND 
+         -- Only DRS files with 100 events and the correct ROI are considered
+         R2.ZNAXIS2=1000 AND R1.NROI=R2.NROI 
+	 AND R1.NIGHT = ${NIGHT}
+	 AND R1.NIGHT = ${NIGHT}
+	 AND R1.Telescope = ${TEL}
+      -- AND
+         -- Check if they are valid
+         -- IFNULL(DoNotUse.WrongVoltage,false)!=true
+      -- This can be used to restrict the number of combinations     
+      -- AND ABS(UNIX_TIMESTAMP(R2.\`DATE-OBS\`)-UNIX_TIMESTAMP(R1.\`DATE-OBS\`))<78*3600
+      ORDER BY
+         R1.Telescope,
+         R1.NIGHT,
+         R1.RUNID,
+         DeltaT
+    )
+    SELECT
+       Telescope,
+       NIGHT,
+       RUNID,
+       DrsNight,
+       DrsRunId,
+       DeltaT/3600e0 AS DeltaT
+    FROM
+       Table1
+    WHERE
+       -- Select only the rows with the smallest DeltaT
+       RowNumber = 1
+);
+
+${INSERT} Calibration SELECT * FROM Calibration_tmp;
+"\ >> ${OUTPUT}