Changeset 20106 for trunk/Mars

Timestamp:

09/07/21 14:03:51 (3 years ago)

Author:

maslowski

Message:

Fixed some bugs.

Location:

trunk/Mars/hawc/processing/DiskToDB

Files:

• run-scripts.sh (modified) (3 diffs)
• update-aux-table.sh (modified) (2 diffs)
• update-file-table2.sh (modified) (1 diff)
• write-calibration-query.sh (modified) (1 diff)

trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh

@@ -229 +229 @@
 mkdir -p ./queries
 readonly ONDISK="./queries/insert-OnDisk-${TEL}.sql"
+readonly AUXDISK="./queries/insert-AuxOnDisk-${TEL}.sql"
 readonly RAW="./queries/insert-raw-${TEL}.sql"
 readonly CALIB="./queries/find-calibration2-${TEL}.sql"
@@ -239 +240 @@
 
 # Create Database of existing files
-# Create SQL query with:
-# (Telescope, Night, RUNID, ISDRSFILE).
 echo "Updating file table"
 bash ./update-file-table2.sh -i -o "${ONDISK}" "${TEL}" "${DIR}/raw"
@@ -246 +245 @@
 echo "Uploading updated table"
 mysql --defaults-file="${CRED}" < "${ONDISK}"
+
+# Create Database of existing auxiliary files
+bash ./update-aux-table.sh -r -o "${AUXDISK}" "${TEL}" "${DIR}/auxil"
+# Update DataOnDisk tabel in database
+echo "Uploading updated table"
+mysql --defaults-file="${CRED}" < "${AUXDISK}"
 
 # get list of NIGHTS with unprocessed files

trunk/Mars/hawc/processing/DiskToDB/update-aux-table.sh

@@ -121 +121 @@
 
 
-echo \
-"\
+        EXT=0
+        NIGHT=`find "${DIR}"/[0-9][0-9][0-9][0-9]/ -type f \
+                | grep -o "/[0-9]\{8\}.${AUX}.fits" \
+                | cut -f2 -d "/" \
+                | cut -f1 -d "."`|| EXT=1
+
+        if [ ${EXT} = "0" ]
+        then
+
+                echo \
+                "\
 INSERT INTO AuxOnDisk_tmp (Telescope, NIGHT, ${AUX})
 VALUES
@@ -128 +137 @@
 
 
-NIGHT=`find "${DIR}"/[0-9][0-9][0-9][0-9]/ -type f \
-        | grep -o "/[0-9]\{8\}.${AUX}.fits" \
-        | cut -f2 -d "/" \
-        | cut -f1 -d "."`
-
-# Format to (Telescope, NIGHT, FAD_CONTROL_TEMPERATURE),
-QUERY=`paste <(for i in ${NIGHT}; do echo ${TEL}; done) \
-        <(echo "${NIGHT}") \
-        <(for i in ${NIGHT}; do echo "0"; done) \
-        --delimiters "," \
-        | sed 's/^/(/' \
-        | sed 's/$/),/'`
-
-# Write query to OUTPUT file
-echo "${QUERY}" >> ${OUTPUT}
-# Change last comma to semicolon
-sed -i '$s/,$//' "${OUTPUT}"
-
-echo "ON DUPLICATE KEY UPDATE ${AUX} = 0;" >> ${OUTPUT}
-
-# 1) If there is data in the database that is not on disk then set 
-#       ${AUX} = NULL for that night
-# 2) Delete all entries from the database that are not on disk
-# 3) ${INSERT} all entries into the database that are on disk
-echo \
-"\
-CREATE TEMPORARY TABLE TableNotDisk_${AUX}
-(
-        Telescope tinyint UNSIGNED NOT NULL,
-        NIGHT int UNSIGNED NOT NULL,
-        FAD_CONTROL_TEMPERATURE tinyint UNSIGNED,
-        BIAS_CONTROL_DYNAMIC_DATA tinyint UNSIGNED,
-        FTM_CONTROL_DATA tinyint UNSIGNED,
-        FTU_CONTROL_DATA tinyint UNSIGNED,
-        Updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 
-                ON UPDATE CURRENT_TIMESTAMP,
-        PRIMARY KEY (Telescope, NIGHT)
-) SELECT * FROM AuxOnDisk
-        WHERE (Telescope, NIGHT) NOT IN 
-        (SELECT Telescope, NIGHT FROM AuxOnDisk_tmp)
-        AND Telescope=${TEL};
-
-UPDATE AuxOnDisk SET ${AUX} = NULL WHERE NIGHT IN 
-        (SELECT NIGHT FROM TableNotDisk_${AUX})
-        AND Telescope=${TEL};
-
-DELETE FROM AuxOnDisk 
-        WHERE (Telescope, NIGHT) NOT IN 
-        (SELECT Telescope, NIGHT FROM AuxOnDisk_tmp)
-        AND Telescope=${TEL};
-
-${INSERT} AuxOnDisk (SELECT * FROM AuxOnDisk_tmp);
-"\ >> ${OUTPUT}
+                # Format to (Telescope, NIGHT, FAD_CONTROL_TEMPERATURE),
+                QUERY=`paste <(for i in ${NIGHT}; do echo ${TEL}; done) \
+                        <(echo "${NIGHT}") \
+                        <(for i in ${NIGHT}; do echo "0"; done) \
+                        --delimiters "," \
+                        | sed 's/^/(/' \
+                        | sed 's/$/),/'`
+
+                # Write query to OUTPUT file
+                echo "${QUERY}" >> ${OUTPUT}
+                # Change last comma to semicolon
+                sed -i '$s/,$//' "${OUTPUT}"
+
+                echo "ON DUPLICATE KEY UPDATE ${AUX} = 0;" >> ${OUTPUT}
+
+                # 1) If there is data in the database that is not on disk then set 
+                #       ${AUX} = NULL for that night
+                # 2) Delete all entries from the database that are not on disk
+                # 3) ${INSERT} all entries into the database that are on disk
+                echo \
+                "\
+                CREATE TEMPORARY TABLE TableNotDisk_${AUX}
+                (
+                        Telescope tinyint UNSIGNED NOT NULL,
+                        NIGHT int UNSIGNED NOT NULL,
+                        FAD_CONTROL_TEMPERATURE tinyint UNSIGNED,
+                        BIAS_CONTROL_DYNAMIC_DATA tinyint UNSIGNED,
+                        FTM_CONTROL_DATA tinyint UNSIGNED,
+                        FTU_CONTROL_DATA tinyint UNSIGNED,
+                        Updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 
+                                ON UPDATE CURRENT_TIMESTAMP,
+                        PRIMARY KEY (Telescope, NIGHT)
+                ) SELECT * FROM AuxOnDisk
+                        WHERE (Telescope, NIGHT) NOT IN 
+                        (SELECT Telescope, NIGHT FROM AuxOnDisk_tmp)
+                        AND Telescope=${TEL};
+
+                UPDATE DataOnDisk SET auxiliary = NULL
+                        WHERE DataOnDisk.NIGHT IN 
+                        (SELECT NIGHT FROM AuxOnDisk_tmp)
+                        AND Telescope=${TEL};
+
+                DELETE FROM AuxOnDisk 
+                        WHERE (Telescope, NIGHT) NOT IN 
+                        (SELECT Telescope, NIGHT FROM AuxOnDisk_tmp)
+                        AND Telescope=${TEL};
+
+                ${INSERT} AuxOnDisk (SELECT * FROM AuxOnDisk_tmp);
+                "\ >> ${OUTPUT}
+
+        else
+                # 1) If there is data in the database that is not on disk then set 
+                #       ${AUX} = NULL for that night
+                # 2) Delete all entries from the database that are not on disk
+                # 3) ${INSERT} all entries into the database that are on disk
+                echo \
+                "\
+                CREATE TEMPORARY TABLE TableNotDisk_${AUX}
+                (
+                        Telescope tinyint UNSIGNED NOT NULL,
+                        NIGHT int UNSIGNED NOT NULL,
+                        FAD_CONTROL_TEMPERATURE tinyint UNSIGNED,
+                        BIAS_CONTROL_DYNAMIC_DATA tinyint UNSIGNED,
+                        FTM_CONTROL_DATA tinyint UNSIGNED,
+                        FTU_CONTROL_DATA tinyint UNSIGNED,
+                        Updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 
+                                ON UPDATE CURRENT_TIMESTAMP,
+                        PRIMARY KEY (Telescope, NIGHT)
+                ) SELECT * FROM AuxOnDisk
+                        WHERE (Telescope, NIGHT) NOT IN 
+                        (SELECT Telescope, NIGHT FROM AuxOnDisk_tmp)
+                        AND Telescope=${TEL};
+
+                UPDATE DataOnDisk SET auxiliary = NULL
+                        WHERE DataOnDisk.NIGHT IN 
+                        (SELECT NIGHT FROM AuxOnDisk_tmp)
+                        AND Telescope=${TEL};
+
+                DELETE FROM AuxOnDisk 
+                        WHERE (Telescope, NIGHT) NOT IN 
+                        (SELECT Telescope, NIGHT FROM AuxOnDisk_tmp)
+                        AND Telescope=${TEL};
+
+                ${INSERT} AuxOnDisk (SELECT * FROM AuxOnDisk_tmp);
+                "\ >> ${OUTPUT}
+
+
+                echo "No Auxiliary files."
+                continue;
+        fi
+
+
 
 done

trunk/Mars/hawc/processing/DiskToDB/update-file-table2.sh

@@ -147 +147 @@
 sed -i '$s/,$/;/' "${OUTPUT}"
 
-# 1) If there is data in the database that is not on disk then set star = 0
-#    for that night
+# 1) If there is data in the database that is not on disk then set 
+#       header = NULL for that night
 # 2) Delete all entries from the database that are not on disk
 # 3) ${INSERT} all entries into the database that are on disk

trunk/Mars/hawc/processing/DiskToDB/write-calibration-query.sh

@@ -218 +218 @@
         WHERE NIGHT=${NIGHT}
         AND Telescope=${TEL}
-        ) = 0;
+        ) = 0
+        AND NIGHT=${NIGHT}
+        AND Telescope=${TEL};
 "\ >> ${OUTPUT}