source: trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh@ 20095

#!/bin/bash
#
# Synchronizes the HAWC's Eye data on disk with the SQL database.
#
# The following scripts must be in the same directory as this script and 
# the named variables must be eddited in those files.
#       run-scripts.sh (this script)
#               TEL
#               DIR
#               CRED
#               CALLISTORC
#               STARRC
#       update-file-table2.sh
#       extract-raw-header2.sh
#               FITSDUMP
#       write-calibration-query.sh
#       extract-aux-data2.sh
#               MARS
#               MACROS
#       run-callisto2.sh
#               MARS
#               MACROS
#       run-star2.sh
#               MARS
#               MACROS
#       run-root2sql2.sh
#               ROOT2SQL
# 
# Example:
#       bash run-scripts.sh

set -o errexit
set -o errtrace
set -o nounset
set -o pipefail


# Filename
readonly SCRPTN=`basename $0 .sh`

# For lockfile
readonly LOCK="/tmp/${SCRPTN}.lock"

function LockExit()
{
        echo "ERROR: One instance of ${SCRPTN} already running" >&2
        exit 1
}

function DescExit()
{
        echo "ERROR: Filedescriptor 200 already taken" >&2
        exit 1
}

function ErrExit()
{
        echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
        exit 1
}

function StrgCExit()
{
        echo " "
        echo "$0 was forcefully terminated" >&2
        exit 1
}

trap ErrExit ERR
trap StrgCExit INT

exec 200>${LOCK} || DescExit
flock -n 200 || LockExit

# For logfile
mkdir -p log
readonly DATE=`date +%Y%m%d_%H%M`
readonly LOG="./log/${SCRPTN}_${DATE}.log"
exec 3>&1 1>>${LOG}
exec 4>>${LOG}
# exec 3>&1
# exec 4>&1

echo "Start processing data"

function DeleteEntries()
{
        # Delete entries form table ${3} for a given Telescope and Night
        # Arguments: Credentials Telescope Table Night
        mysql --defaults-file="${1}" \
                --skip-column-names \
                --batch \
                --raw \
                --compress \
                -e "DELETE FROM ${3} WHERE Telescope = ${2} AND NIGHT = ${4};"
}

function SetStatus()
{
        # Query with process update
        # Arguments: Credentials Telescope Status Night
        mysql --defaults-file="${1}" \
                --skip-column-names \
                --batch \
                --raw \
                --compress \
                -e "UPDATE DataOnDisk SET STATUS = ${3} WHERE NIGHT = ${4} \
                AND Telescope = ${2};"
}

function GetStatus()
{
        # Return status of Night from database
        # Arguments: Credentials Telescope Night
        local STATUS=`mysql --defaults-file="${1}" \
                --skip-column-names \
                --batch \
                --raw \
                --compress \
                -e "SELECT DISTINCT STATUS FROM DataOnDisk WHERE NIGHT = ${3} \
                AND Telescope = ${2};"`

        if [ -z ${STATUS} ]
        then
                echo "ERROR: Status does not exist for telescope: ${2} and night: ${3}" >&2
                exit 1
        fi

        if [ `echo "${STATUS}" | wc -l` -ne "1" ]
        then
                echo "ERROR: Not all status values are the same for the night: ${3}!" >&2
                exit 1
        fi

        echo "${STATUS}"
}

# Define variables
# Telescope for which to process
readonly TEL=1
# Directory in which the raw data is located
readonly DIR='/run/media/frankm/Neuer Datenträger/data/DATA/HE0'${TEL}
# Credentials for the SQL database
readonly CRED='../credentials-read-only.cnf'
# Credentials for root2sql for callisto
readonly CALLISTORC='../root2sql-callisto.rc'
# Credentials for root2sql for star
readonly STARRC='../root2sql-star.rc'
# SQL outputs
mkdir -p ./queries
readonly ONDISK='./queries/insert-OnDisk.sql'
readonly RAW='./queries/insert-raw.sql'
readonly CALIB='./queries/find-calibration2.sql'
readonly AUX='./queries/insert-aux.sql'

# Create Database of existing files
# Create SQL query with:
# (Telescope, Night, RUNID, ISDRSFILE, ISPROCESSED).
echo "Updating file table"
bash ./update-file-table2.sh -i -o "${ONDISK}" "${TEL}" "${DIR}/raw"
# Update DataOnDisk tabel in database
echo "Uploading updated table"
mysql --defaults-file="${CRED}" < "${ONDISK}"

# get list of NIGHTS with unprocessed files
echo "Getting nights to be processed"
NIGHTS=`mysql --defaults-file="${CRED}" \
        --skip-column-names \
        --batch \
        --raw \
        --compress \
        -e "SELECT DISTINCT NIGHT FROM DataOnDisk WHERE STATUS != 5 \
        AND Telescope = ${TEL};"`

# Check if NIGHTS is empty and dont do anythin if so.
if [ -z "${NIGHTS}" ]
then
        echo "Database is up to date. Nothing to do here."
        exit 0
fi

echo "Updating database"

echo "Nights to process:"
echo "${NIGHTS}"
echo " "


# Execute scripts on all RunId for given Telescope and Night that has
# Loop over nights and upload to database for every loop
for NIGHT in ${NIGHTS}
do
        STATUS=`GetStatus ${CRED} ${TEL} ${NIGHT}`
        echo "Status = ${STATUS} for night = ${NIGHT}"
        echo " "

        if [ ${STATUS} -lt "1" ]
        then
                # Get raw data headers
                echo "== In Status 1 =="
                bash ./extract-raw-header2.sh -d -o "${RAW}" \
                        ${TEL} \
                        "${DIR}"/raw \
                        ${NIGHT}
                echo "Uploading raw headers for night: ${NIGHT}"
                mysql --defaults-file="${CRED}" < "${RAW}"
                SetStatus ${CRED} ${TEL} "1" ${NIGHT}
                echo "Finished uploading raw headers for night: ${NIGHT}"
                echo " "
        fi

        if [ ${STATUS} -lt "2" ]
        then
                # Get calibration files
                echo "== In Status 2 =="
                bash ./write-calibration-query.sh -d -o "${CALIB}" \
                        ${TEL} \
                        ${NIGHT}
                echo "Uploading calibration for night: ${NIGHT}"
                mysql --defaults-file="${CRED}" < "${CALIB}"
                SetStatus ${CRED} ${TEL} "2" ${NIGHT}
                echo "Finished uploading calibration for night: ${NIGHT}"
                echo " "
        fi

        if [ ${STATUS} -lt "3" ]
        then
                # Get aux data
                echo "== In Status 3 =="
                bash ./extract-aux-data2.sh -d -o "${AUX}" \
                        ${CRED} \
                        ${TEL} \
                        "${DIR}"/auxil \
                        ${NIGHT}
                echo "Uploading aux data for night: ${NIGHT}"
                mysql --defaults-file="${CRED}" < "${AUX}"
                SetStatus ${CRED} ${TEL} "3" ${NIGHT}
                echo "Finished uploading aux data for night: ${NIGHT}"
                echo " "
        fi

        if [ ${STATUS} -lt "4" ]
        then
                # Get callisto data
                echo "== In Status 4 =="
                bash ./run-callisto2.sh -r \
                        "${CRED}" \
                        ${TEL} \
                        "${DIR}"/raw \
                        ${NIGHT}

                echo "Deleting callisto entries for night: ${NIGHT}"
                DeleteEntries ${CRED} ${TEL} "Callisto" ${NIGHT}

                echo "Uploading callisto entries for night: ${NIGHT}"
                bash ./run-root2sql2.sh -Y \
                        "${CRED}" \
                        "${CALLISTORC}" \
                        ${TEL} \
                        "${DIR}"/callisto \
                        ${NIGHT}
                SetStatus ${CRED} ${TEL} "4" ${NIGHT}
                echo "Finished uploading callisto entries for night: ${NIGHT}"
                echo " "
        fi

        if [ ${STATUS} -lt "5" ]
        then
                # Get star data
                echo "== In Status 5 =="
                bash ./run-star2.sh -r "${CRED}" ${TEL} "${DIR}"/raw ${NIGHT}

                echo "Deleting star entries for night: ${NIGHT}"
                DeleteEntries ${CRED} ${TEL} "Star" ${NIGHT}

                echo "Uploading star entries for night: ${NIGHT}"
                bash ./run-root2sql2.sh -I \
                        "${CRED}" \
                        "${STARRC}" \
                        ${TEL} \
                        "${DIR}"/star \
                        ${NIGHT}
                SetStatus ${CRED} ${TEL} "5" ${NIGHT}
                echo "Finished uploading star entries for night: ${NIGHT}"
                echo " "
        fi

done

echo " "
echo "Finished processing data"