Changeset 20103 for trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh

Timestamp:

09/01/21 07:56:42 (3 years ago)

Author:

maslowski

Message:

Added better error handeling and tracking in the databas.

File:

• trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh (modified) (12 diffs)

trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh

@@ -77 +77 @@
 readonly DATE=`date +%Y%m%d_%H%M`
 readonly LOG="./log/${SCRPTN}_${DATE}.log"
-exec 3>&1 1>>${LOG}
-exec 4>>${LOG}
-# exec 3>&1
-# exec 4>&1
+# exec 3>&1 1>>${LOG}
+# exec 4>>${LOG}
+exec 3>&1
+exec 4>&1
 
 echo "Start processing data"
@@ -88 +88 @@
         # Delete entries form table ${3} for a given Telescope and Night
         # Arguments: Credentials Telescope Table Night
-        mysql --defaults-file="${1}" \
+        local cred="${1}"
+        local tel="${2}"
+        local table="${3}"
+        local night="${4}"
+
+        mysql --defaults-file="${cred}" \
                 --skip-column-names \
                 --batch \
                 --raw \
                 --compress \
-                -e "DELETE FROM ${3} WHERE Telescope = ${2} AND NIGHT = ${4};"
+                -e "DELETE FROM ${table} WHERE Telescope = ${tel} \
+                AND NIGHT = ${night};"
 }
 
 function SetStatus()
 {
-        # Query with process update
-        # Arguments: Credentials Telescope Status Night
-        mysql --defaults-file="${1}" \
+        # Status update for one Column, Night and Telescope
+        # Arguments: Credentials Telescope Column Value Night
+        local cred="${1}"
+        local tel="${2}"
+        local col="${3}"
+        local val="${4}"
+        local night="${5}"
+
+        # Check if a valid Column was selected
+        if [ ${col} != "header" ] \
+                && [ ${col} != "calibration" ] \
+                && [ ${col} != "auxiliary" ] \
+                && [ ${col} != "callisto" ] \
+                && [ ${col} != "star" ]
+        then
+                echo "ERROR: ${col} is not a valid column."
+                exit 1
+        fi
+
+        mysql --defaults-file="${cred}" \
                 --skip-column-names \
                 --batch \
                 --raw \
                 --compress \
-                -e "UPDATE DataOnDisk SET STATUS = ${3} WHERE NIGHT = ${4} \
-                AND Telescope = ${2};"
+                -e "UPDATE DataOnDisk SET ${col} = ${val}\
+                WHERE NIGHT = ${night} AND Telescope = ${tel};"
 }
 
 function GetStatus()
 {
-        # Return status of Night from database
-        # Arguments: Credentials Telescope Night
-        local STATUS=`mysql --defaults-file="${1}" \
+        # Return status of Night for a given Column and Telescope
+        # Arguments: Credentials Telescope Column Night
+        local cred="${1}"
+        local tel="${2}"
+        local col="${3}"
+        local night="${4}"
+
+        # Check if a valid Column was selected
+        if [ ${col} != "header" ] \
+                && [ ${col} != "calibration" ] \
+                && [ ${col} != "auxiliary" ] \
+                && [ ${col} != "callisto" ] \
+                && [ ${col} != "star" ]
+        then
+                echo "ERROR: ${col} is not a valid column."
+                exit 1
+        fi
+
+        local STATUS=`mysql --defaults-file="${cred}" \
                 --skip-column-names \
                 --batch \
                 --raw \
                 --compress \
-                -e "SELECT DISTINCT STATUS FROM DataOnDisk WHERE NIGHT = ${3} \
-                AND Telescope = ${2};"`
-
-        if [ -z ${STATUS} ]
-        then
-                echo "ERROR: Status does not exist for telescope: ${2} and night: ${3}" >&2
+                -e "SELECT DISTINCT ${col} FROM DataOnDisk \
+                WHERE NIGHT = ${night} \
+                AND Telescope = ${tel} \
+                AND ${col} IS NULL;"`
+
+        echo "${STATUS}"
+}
+
+
+function DoesDirExists()
+{
+        local thing="${1}"
+        # Check if thing exists
+        if [ ! -d "${thing}" ]
+        then
+                echo "${thing} is not a valid Path. Please change variable in script ${SCRPTN}."
                 exit 1
         fi
-
-        if [ `echo "${STATUS}" | wc -l` -ne "1" ]
-        then
-                echo "ERROR: Not all status values are the same for the night: ${3}!" >&2
+}
+
+function DoesFileExists()
+{
+        local thing="${1}"
+        # Check if thing exists
+        if [ ! -f "${thing}" ]
+        then
+                echo "${thing} is not a valid File. Please change variable in script ${SCRPTN}."
                 exit 1
         fi
-
-        echo "${STATUS}"
-}
+}
+
 
 # Define variables
@@ -147 +200 @@
 # Credentials for root2sql for star
 readonly STARRC='../root2sql-star.rc'
+
+# Check if variable paths point to something sensible
+DoesDirExists "${DIR}"
+DoesFileExists "${CRED}"
+DoesFileExists "${CALLISTORC}"
+DoesFileExists "${STARRC}"
+
 # SQL outputs
 mkdir -p ./queries
@@ -156 +216 @@
 # Create Database of existing files
 # Create SQL query with:
-# (Telescope, Night, RUNID, ISDRSFILE, ISPROCESSED).
+# (Telescope, Night, RUNID, ISDRSFILE).
 echo "Updating file table"
 bash ./update-file-table2.sh -i -o "${ONDISK}" "${TEL}" "${DIR}/raw"
@@ -170 +230 @@
         --raw \
         --compress \
-        -e "SELECT DISTINCT NIGHT FROM DataOnDisk WHERE STATUS != 5 \
+        -e "SELECT DISTINCT NIGHT FROM DataOnDisk \
+        WHERE header IS NULL \
+        OR calibration IS NULL \
+        OR auxiliary IS NULL \
+        OR callisto IS NULL \
+        OR star IS NULL \
         AND Telescope = ${TEL};"`
 
@@ -182 +247 @@
 echo "Updating database"
 
+NIGHTS=20201111
+
 echo "Nights to process:"
 echo "${NIGHTS}"
@@ -191 +258 @@
 for NIGHT in ${NIGHTS}
 do
-        STATUS=`GetStatus ${CRED} ${TEL} ${NIGHT}`
-        echo "Status = ${STATUS} for night = ${NIGHT}"
-        echo " "
-
-        if [ ${STATUS} -lt "1" ]
+        HEAD_S=`GetStatus ${CRED} ${TEL} "header" ${NIGHT}`
+        CALI_S=`GetStatus ${CRED} ${TEL} "calibration" ${NIGHT}`
+        AUXI_S=`GetStatus ${CRED} ${TEL} "auxiliary" ${NIGHT}`
+        CALL_S=`GetStatus ${CRED} ${TEL} "callisto" ${NIGHT}`
+        STAR_S=`GetStatus ${CRED} ${TEL} "star" ${NIGHT}`
+
+        echo "header = ${HEAD_S} for night = ${NIGHT}"
+        echo " "
+
+        # If header is null
+        if [ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]
         then
                 # Get raw data headers
@@ -205 +278 @@
                 echo "Uploading raw headers for night: ${NIGHT}"
                 mysql --defaults-file="${CRED}" < "${RAW}"
-                SetStatus ${CRED} ${TEL} "1" ${NIGHT}
                 echo "Finished uploading raw headers for night: ${NIGHT}"
                 echo " "
         fi
 
-        if [ ${STATUS} -lt "2" ]
+        echo "calibration = ${CALI_S} for night = ${NIGHT}"
+        echo " "
+
+        # If header is null and or calibration is null
+        if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
+                || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ])
         then
                 # Get calibration files
@@ -219 +296 @@
                 echo "Uploading calibration for night: ${NIGHT}"
                 mysql --defaults-file="${CRED}" < "${CALIB}"
-                SetStatus ${CRED} ${TEL} "2" ${NIGHT}
                 echo "Finished uploading calibration for night: ${NIGHT}"
                 echo " "
         fi
 
-        if [ ${STATUS} -lt "3" ]
+        echo "auxiliary = ${AUXI_S} for night = ${NIGHT}"
+        echo " "
+
+        # If header is null and or calibration is null and or auxiliary is null
+        if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
+                || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
+                || ([ ! -z ${AUXI_S} ] && [ ${AUXI_S} = "NULL" ])
         then
                 # Get aux data
@@ -235 +317 @@
                 echo "Uploading aux data for night: ${NIGHT}"
                 mysql --defaults-file="${CRED}" < "${AUX}"
-                SetStatus ${CRED} ${TEL} "3" ${NIGHT}
                 echo "Finished uploading aux data for night: ${NIGHT}"
                 echo " "
         fi
 
-        if [ ${STATUS} -lt "4" ]
+        echo "callisto = ${CALL_S} for night = ${NIGHT}"
+        echo " "
+
+        # If header is null and or calibration is null and or callisto is null
+        if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
+                || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
+                || ([ ! -z ${CALL_S} ] && [ ${CALL_S} = "NULL" ])
         then
                 # Get callisto data
@@ -260 +347 @@
                         "${DIR}"/callisto \
                         ${NIGHT}
-                SetStatus ${CRED} ${TEL} "4" ${NIGHT}
                 echo "Finished uploading callisto entries for night: ${NIGHT}"
                 echo " "
         fi
 
-        if [ ${STATUS} -lt "5" ]
+        echo "star = ${STAR_S} for night = ${NIGHT}"
+        echo " "
+
+        # If header is null and or calibration is null and or callisto is null
+        # and or star is null
+        if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
+                || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
+                || ([ ! -z ${CALL_S} ] && [ ${CALL_S} = "NULL" ]) \
+                || ([ ! -z ${STAR_S} ] && [ ${STAR_S} = "NULL" ])
         then
                 # Get star data
@@ -281 +375 @@
                         "${DIR}"/star \
                         ${NIGHT}
-                SetStatus ${CRED} ${TEL} "5" ${NIGHT}
                 echo "Finished uploading star entries for night: ${NIGHT}"
                 echo " "