source: trunk/Mars/hawc/processing/DiskToDB/extract-raw-header2.sh@ 20103

#!/bin/bash

# Read fits files from DIR for a given telescope TEL and night NIGHT and write 
# an SQL query to OUTPUT for uploading the raw header information. 
#
# Example:
#       bash extract-raw-header2.sh 1 /data/HE01/raw 20201111
#

set -o errexit
set -o errtrace
set -o nounset
set -o pipefail

function ErrExit()
{
        echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
        exit 1
}

function StrgCExit()
{
        echo " "
        echo "$0 was forcefully terminated" >&2
        exit 1
}

trap ErrExit ERR
trap StrgCExit INT


# define variables
OUTPUT=insert-raw.sql
readonly FITSDUMP="/home/frankm/Dateien/FACT++/build/fitsdump"
readonly PROGRAM=$0
INSERT="INSERT INTO"

# Check if fitsdump exists
if [ ! -f "${FITSDUMP}" ]
then
        echo "ERROR: fitsdump does not exists at ${FITSDUMP}. Please change in script ${PROGRAM}." >&2
        exit 1
fi

usage()
{
        echo "usage: $PROGRAM [-hird] [-o outfile] [Telescope Directory Night]"
        echo "  -h              display help"
        echo "  -i              query ignores already existing rows"
        echo "  -r              query replaces already existing rows (overwrites -i)"
        echo "  -d              delete all entries for a given Night and Telescope"
        echo "  -o outfile      name of the SQL query file"
        exit 1;
}

# Check for flags
while getopts 'hirdo:' flag
do
        case "${flag}" in
                h) usage ;;
                i) INSERT="INSERT IGNORE" ;;
                r) INSERT="REPLACE INTO" ;;
                d) DELETE="true" ;;
                o) OUTPUT="${OPTARG}" ;;
                *) usage ;;
        esac
done
shift $(($OPTIND-1))

# Check if at least two arguments are provided
if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ]
then
        echo "ERROR: Not enough arguments. Check -h for help!" >&2
        exit 1
fi

TEL="${1}"
DIR="${2}"
NIGHT="${3}"

readonly SUBDIR="${DIR}"/${NIGHT:0:4}/${NIGHT:4:2}/${NIGHT:6:2}

# Keywords to grep from fitsdump
readonly TOGREP='NIGHT\|RUNID\|TSTART\|TSTOP\|STEP\|DATE\|NROI[^T]\|DAC[0-7]\|DNA[01][^0-9]\|FWVER[01][^0-9]\|DRSCALIB\|NTRG\|NTRG\|ZNAXIS2\|REFCLK\|ZRATIO\|RUNTYPE'
# Refference declarations for checking the header formatting
readonly SEARCHSHORT='DAC0\|DAC1\|DAC2\|DAC3\|DAC4\|DAC5\|DAC6\|DAC7\|`DATE`\|`DATE-END`\|`DATE-OBS`\|DNA0\|DNA1\|DRSCALIB\|FWVER0\|FWVER1\|NIGHT\|NROI\|NTRG\|NTRGEXT1\|NTRGEXT2\|NTRGLPE\|NTRGLPI\|NTRGMISC\|NTRGPED\|NTRGTIM\|REFCLK\|RUNID\|RUNTYPE\|TSTARTF\|TSTARTI\|TSTOPF\|TSTOPI\|ZNAXIS2\|ZRATIO'
readonly SEARCHLONG='DAC0\|DAC1\|DAC2\|DAC3\|DAC4\|DAC5\|DAC6\|DAC7\|`DATE`\|`DATE-END`\|`DATE-OBS`\|DNA0\|DNA1\|DRSCALIB\|DRSSTEP\|FWVER0\|FWVER1\|NIGHT\|NROI\|NTRG\|NTRGEXT1\|NTRGEXT2\|NTRGLPE\|NTRGLPI\|NTRGMISC\|NTRGPED\|NTRGTIM\|REFCLK\|RUNID\|RUNTYPE\|TSTARTF\|TSTARTI\|TSTOPF\|TSTOPI\|ZNAXIS2\|ZRATIO'

readonly REFSHORT=`echo "${SEARCHSHORT}" | grep -o "${SEARCHSHORT}"`
readonly REFLONG=`echo "${SEARCHLONG}" | grep -o "${SEARCHLONG}"`
readonly CNTSHORT=34
readonly CNTLONG=35

function InsertRaw()
{
        local res="${1}"

        echo "${INSERT} RawData SET"            >> ${OUTPUT}
        echo Telescope=${TEL},                  >> ${OUTPUT}
        echo "${res}"                           >> ${OUTPUT}
        echo ";"                                >> ${OUTPUT}
}

function SetRawStatus()
{
        local root="${1}"
        local runid="${2}"
        local stat="${3}"

        echo "${root} header not valid!"
        echo "UPDATE DataOnDisk SET"            >> ${OUTPUT}
        echo "header=${stat} "                  >> ${OUTPUT}
        echo "WHERE NIGHT = ${NIGHT}"           >> ${OUTPUT}
        echo "AND Telescope = ${TEL}"           >> ${OUTPUT}
        echo "AND RUNID = ${runid}"             >> ${OUTPUT}
        echo ";"                                >> ${OUTPUT}
}

echo \
"\
CREATE TABLE IF NOT EXISTS RawData
(
        Telescope tinyint UNSIGNED NOT NULL,
        NIGHT int UNSIGNED NOT NULL,
        RUNID mediumint UNSIGNED NOT NULL,
        \`DATE-OBS\` datetime(6) NOT NULL,
        \`DATE-END\` datetime(6) NOT NULL,
        RUNTYPE tinyint UNSIGNED NOT NULL,
        DRSSTEP tinyint UNSIGNED DEFAULT NULL,
        NROI mediumint UNSIGNED NOT NULL,
        ZNAXIS2 bigint UNSIGNED NOT NULL,
        NTRG int UNSIGNED NOT NULL,
        NTRGMISC int UNSIGNED NOT NULL,
        NTRGPED int UNSIGNED NOT NULL,
        REFCLK float NOT NULL,
        ZRATIO float NOT NULL,
        DAC0 mediumint UNSIGNED NOT NULL,
        DAC1 mediumint UNSIGNED NOT NULL,
        DAC2 mediumint UNSIGNED NOT NULL,
        DAC3 mediumint UNSIGNED NOT NULL,
        DAC4 mediumint UNSIGNED NOT NULL,
        DAC5 mediumint UNSIGNED NOT NULL,
        DAC6 mediumint UNSIGNED NOT NULL,
        DAC7 mediumint UNSIGNED NOT NULL,
        \`DATE\` timestamp NOT NULL,
        DRSCALIB tinyint(1) NOT NULL,
        NTRGEXT1 int UNSIGNED NOT NULL,
        NTRGEXT2 int UNSIGNED NOT NULL,
        NTRGLPE int UNSIGNED NOT NULL,
        NTRGLPI int UNSIGNED NOT NULL,
        NTRGTIM int UNSIGNED NOT NULL,
        TSTARTF double NOT NULL,
        TSTARTI mediumint UNSIGNED NOT NULL,
        TSTOPF double NOT NULL,
        TSTOPI mediumint UNSIGNED NOT NULL,
        DNA0 bigint UNSIGNED NOT NULL,
        DNA1 bigint UNSIGNED NOT NULL,
        FWVER0 mediumint UNSIGNED NOT NULL,
        FWVER1 mediumint UNSIGNED NOT NULL,
        PRIMARY KEY (Telescope, NIGHT, RUNID),
        KEY (RUNTYPE)
); 
"\ > ${OUTPUT}

if [ ${DELETE} = "true" ]
then
        echo "DELETE FROM RawData" >> ${OUTPUT}
        echo "WHERE Telescope = ${TEL}" >> ${OUTPUT}
        echo "AND NIGHT = ${NIGHT};" >> ${OUTPUT}
        echo "" >> ${OUTPUT}
fi


# Loop over all input arguments
for ROOT in "${SUBDIR}"/* 
do

        # check if file extension is correct
        if [ "${ROOT: -8}" == ".fits.fz" ]
        then
                echo "${ROOT}"
                RUNID=`basename "${ROOT}" .fits.fz | cut -f2 -d "_"`
        else
                echo "${ROOT} has wrong file extension and will not be processed."
                continue
        fi
        EXT=0

        # Extract the header from the fits file with fitsdump, 
        # the following chain properly formats the output, 
        # replaces T/F with true/false, replaces run-ytpes by numeric 
        # types, adds enclosures for SQL variable names where
        # necessary, removes enclosures around hex-numbers and adds 
        # commas after all lines except the last one
        RESULT=`${FITSDUMP} -h "${ROOT}" 2>&4 \
                | grep "${TOGREP}" \
                | grep -v CHECKSUM \
                | cut -c4-  \
                | cut -f1 -d\/ \
                | sed "s/'drs-pedestal'/1/g" \
                | sed "s/'drs-gain'/2/g" \
                | sed "s/'pedestal'/3/g" \
                | sed "s/'data'/4/g" \
                | sed "s/'custom'/0/g" \
                | sed 's/\ T\ /true/g' \
                | sed 's/\ F\ /false/g' \
                | sed 's/^DATE-END/\`DATE-END\`/' \
                | sed 's/^DATE-OBS/\`DATE-OBS\`/' \
                | sed 's/^DATE[^-]/\`DATE\`/' \
                | sed "s/^\(.*\)'\(0x[0-9a-f]*\)'\(.*\)$/\1\2\3/g" \
                | sed '$!s/$/,/'` || EXT=1

        if [ $EXT -ne 0 ]
        then
                SetRawStatus "${ROOT}" ${RUNID} "1"
                continue
        fi

        # compare to refference header and count commas
        CHECKSHORT=`echo ${RESULT} | grep -o "${SEARCHSHORT}"`
        CHECKLONG=`echo ${RESULT} | grep -o "${SEARCHLONG}"`
        CNTCHECK=`echo "${RESULT}" | grep -o ',' | wc -l`

        # If long or short CHECK are matching, write result to file
        if [ "${CHECKLONG}" == "${REFLONG}" ] && 
                [ "${CNTCHECK}" == "${CNTLONG}" ]
        then
                InsertRaw "${RESULT}"
                SetRawStatus "${ROOT}" ${RUNID} "0"
        elif [ "${CHECKSHORT}" == "${REFSHORT}" ] && 
                [ "${CNTCHECK}" == "${CNTSHORT}" ]
        then
                InsertRaw "${RESULT}"
                SetRawStatus "${ROOT}" ${RUNID} "0"
        else
                SetRawStatus "${ROOT}" ${RUNID} "2"
        fi

done