| 1 | #!/bin/bash
|
|---|
| 2 |
|
|---|
| 3 | # Read fits files from DIR for a given telescope TEL and night NIGHT and write
|
|---|
| 4 | # an SQL query to OUTPUT for uploading the raw header information.
|
|---|
| 5 | #
|
|---|
| 6 | # Example:
|
|---|
| 7 | # bash extract-raw-header2.sh 1 /data/HE01/raw 20201111
|
|---|
| 8 | #
|
|---|
| 9 |
|
|---|
| 10 | set -o errexit
|
|---|
| 11 | set -o errtrace
|
|---|
| 12 | set -o nounset
|
|---|
| 13 | set -o pipefail
|
|---|
| 14 |
|
|---|
| 15 | function ErrExit()
|
|---|
| 16 | {
|
|---|
| 17 | echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
|
|---|
| 18 | exit 1
|
|---|
| 19 | }
|
|---|
| 20 |
|
|---|
| 21 | function StrgCExit()
|
|---|
| 22 | {
|
|---|
| 23 | echo " "
|
|---|
| 24 | echo "$0 was forcefully terminated" >&2
|
|---|
| 25 | exit 1
|
|---|
| 26 | }
|
|---|
| 27 |
|
|---|
| 28 | trap ErrExit ERR
|
|---|
| 29 | trap StrgCExit INT
|
|---|
| 30 |
|
|---|
| 31 |
|
|---|
| 32 | # define variables
|
|---|
| 33 | OUTPUT=insert-raw.sql
|
|---|
| 34 | readonly FITSDUMP="/home/hawc/Desktop/FACT++/fitsdump"
|
|---|
| 35 | readonly PROGRAM=$0
|
|---|
| 36 | INSERT="INSERT INTO"
|
|---|
| 37 |
|
|---|
| 38 | # Check if fitsdump exists
|
|---|
| 39 | if [ ! -f "${FITSDUMP}" ]
|
|---|
| 40 | then
|
|---|
| 41 | echo "ERROR: fitsdump does not exists at ${FITSDUMP}. Please change in script ${PROGRAM}." >&2
|
|---|
| 42 | exit 1
|
|---|
| 43 | fi
|
|---|
| 44 |
|
|---|
| 45 | usage()
|
|---|
| 46 | {
|
|---|
| 47 | echo "usage: $PROGRAM [-hird] [-o outfile] [Telescope Directory Night]"
|
|---|
| 48 | echo " -h display help"
|
|---|
| 49 | echo " -i query ignores already existing rows"
|
|---|
| 50 | echo " -r query replaces already existing rows (overwrites -i)"
|
|---|
| 51 | echo " -d delete all entries for a given Night and Telescope"
|
|---|
| 52 | echo " -o outfile name of the SQL query file"
|
|---|
| 53 | exit 1;
|
|---|
| 54 | }
|
|---|
| 55 |
|
|---|
| 56 | # Check for flags
|
|---|
| 57 | while getopts 'hirdo:' flag
|
|---|
| 58 | do
|
|---|
| 59 | case "${flag}" in
|
|---|
| 60 | h) usage ;;
|
|---|
| 61 | i) INSERT="INSERT IGNORE" ;;
|
|---|
| 62 | r) INSERT="REPLACE INTO" ;;
|
|---|
| 63 | d) DELETE="true" ;;
|
|---|
| 64 | o) OUTPUT="${OPTARG}" ;;
|
|---|
| 65 | *) usage ;;
|
|---|
| 66 | esac
|
|---|
| 67 | done
|
|---|
| 68 | shift $(($OPTIND-1))
|
|---|
| 69 |
|
|---|
| 70 | # Check if at least two arguments are provided
|
|---|
| 71 | if [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ]
|
|---|
| 72 | then
|
|---|
| 73 | echo "ERROR: Not enough arguments. Check -h for help!" >&2
|
|---|
| 74 | exit 1
|
|---|
| 75 | fi
|
|---|
| 76 |
|
|---|
| 77 | TEL="${1}"
|
|---|
| 78 | DIR="${2}"
|
|---|
| 79 | NIGHT="${3}"
|
|---|
| 80 |
|
|---|
| 81 | readonly SUBDIR="${DIR}"/${NIGHT:0:4}/${NIGHT:4:2}/${NIGHT:6:2}
|
|---|
| 82 |
|
|---|
| 83 | # Keywords to grep from fitsdump
|
|---|
| 84 | readonly TOGREP='NIGHT\|RUNID\|TSTART\|TSTOP\|STEP\|DATE\|NROI[^T]\|DAC[0-7]\|DNA[01][^0-9]\|FWVER[01][^0-9]\|DRSCALIB\|NTRG\|NTRG\|ZNAXIS2\|REFCLK\|ZRATIO\|RUNTYPE'
|
|---|
| 85 | # Refference declarations for checking the header formatting
|
|---|
| 86 | readonly SEARCHSHORT='DAC0\|DAC1\|DAC2\|DAC3\|DAC4\|DAC5\|DAC6\|DAC7\|`DATE`\|`DATE-END`\|`DATE-OBS`\|DNA0\|DNA1\|DRSCALIB\|FWVER0\|FWVER1\|NIGHT\|NROI\|NTRG\|NTRGEXT1\|NTRGEXT2\|NTRGLPE\|NTRGLPI\|NTRGMISC\|NTRGPED\|NTRGTIM\|REFCLK\|RUNID\|RUNTYPE\|TSTARTF\|TSTARTI\|TSTOPF\|TSTOPI\|ZNAXIS2\|ZRATIO'
|
|---|
| 87 | readonly SEARCHLONG='DAC0\|DAC1\|DAC2\|DAC3\|DAC4\|DAC5\|DAC6\|DAC7\|`DATE`\|`DATE-END`\|`DATE-OBS`\|DNA0\|DNA1\|DRSCALIB\|DRSSTEP\|FWVER0\|FWVER1\|NIGHT\|NROI\|NTRG\|NTRGEXT1\|NTRGEXT2\|NTRGLPE\|NTRGLPI\|NTRGMISC\|NTRGPED\|NTRGTIM\|REFCLK\|RUNID\|RUNTYPE\|TSTARTF\|TSTARTI\|TSTOPF\|TSTOPI\|ZNAXIS2\|ZRATIO'
|
|---|
| 88 |
|
|---|
| 89 | readonly REFSHORT=`echo "${SEARCHSHORT}" | grep -o "${SEARCHSHORT}"`
|
|---|
| 90 | readonly REFLONG=`echo "${SEARCHLONG}" | grep -o "${SEARCHLONG}"`
|
|---|
| 91 | readonly CNTSHORT=34
|
|---|
| 92 | readonly CNTLONG=35
|
|---|
| 93 |
|
|---|
| 94 | function InsertRaw()
|
|---|
| 95 | {
|
|---|
| 96 | local res="${1}"
|
|---|
| 97 |
|
|---|
| 98 | echo "${INSERT} RawData SET" >> ${OUTPUT}
|
|---|
| 99 | echo Telescope=${TEL}, >> ${OUTPUT}
|
|---|
| 100 | echo "${res}" >> ${OUTPUT}
|
|---|
| 101 | echo ";" >> ${OUTPUT}
|
|---|
| 102 | }
|
|---|
| 103 |
|
|---|
| 104 | function SetRawStatus()
|
|---|
| 105 | {
|
|---|
| 106 | local root="${1}"
|
|---|
| 107 | local runid="${2}"
|
|---|
| 108 | local stat="${3}"
|
|---|
| 109 |
|
|---|
| 110 | echo "UPDATE DataOnDisk SET" >> ${OUTPUT}
|
|---|
| 111 | echo "header=${stat} " >> ${OUTPUT}
|
|---|
| 112 | echo "WHERE NIGHT = ${NIGHT}" >> ${OUTPUT}
|
|---|
| 113 | echo "AND Telescope = ${TEL}" >> ${OUTPUT}
|
|---|
| 114 | echo "AND RUNID = ${runid}" >> ${OUTPUT}
|
|---|
| 115 | echo ";" >> ${OUTPUT}
|
|---|
| 116 | }
|
|---|
| 117 |
|
|---|
| 118 | echo \
|
|---|
| 119 | "\
|
|---|
| 120 | CREATE TABLE IF NOT EXISTS RawData
|
|---|
| 121 | (
|
|---|
| 122 | Telescope tinyint UNSIGNED NOT NULL,
|
|---|
| 123 | NIGHT int UNSIGNED NOT NULL,
|
|---|
| 124 | RUNID mediumint UNSIGNED NOT NULL,
|
|---|
| 125 | \`DATE-OBS\` datetime(6) NOT NULL,
|
|---|
| 126 | \`DATE-END\` datetime(6) NOT NULL,
|
|---|
| 127 | RUNTYPE tinyint UNSIGNED NOT NULL,
|
|---|
| 128 | DRSSTEP tinyint UNSIGNED DEFAULT NULL,
|
|---|
| 129 | NROI mediumint UNSIGNED NOT NULL,
|
|---|
| 130 | ZNAXIS2 bigint UNSIGNED NOT NULL,
|
|---|
| 131 | NTRG int UNSIGNED NOT NULL,
|
|---|
| 132 | NTRGMISC int UNSIGNED NOT NULL,
|
|---|
| 133 | NTRGPED int UNSIGNED NOT NULL,
|
|---|
| 134 | REFCLK float NOT NULL,
|
|---|
| 135 | ZRATIO float NOT NULL,
|
|---|
| 136 | DAC0 mediumint UNSIGNED NOT NULL,
|
|---|
| 137 | DAC1 mediumint UNSIGNED NOT NULL,
|
|---|
| 138 | DAC2 mediumint UNSIGNED NOT NULL,
|
|---|
| 139 | DAC3 mediumint UNSIGNED NOT NULL,
|
|---|
| 140 | DAC4 mediumint UNSIGNED NOT NULL,
|
|---|
| 141 | DAC5 mediumint UNSIGNED NOT NULL,
|
|---|
| 142 | DAC6 mediumint UNSIGNED NOT NULL,
|
|---|
| 143 | DAC7 mediumint UNSIGNED NOT NULL,
|
|---|
| 144 | \`DATE\` timestamp NOT NULL,
|
|---|
| 145 | DRSCALIB tinyint(1) NOT NULL,
|
|---|
| 146 | NTRGEXT1 int UNSIGNED NOT NULL,
|
|---|
| 147 | NTRGEXT2 int UNSIGNED NOT NULL,
|
|---|
| 148 | NTRGLPE int UNSIGNED NOT NULL,
|
|---|
| 149 | NTRGLPI int UNSIGNED NOT NULL,
|
|---|
| 150 | NTRGTIM int UNSIGNED NOT NULL,
|
|---|
| 151 | TSTARTF double NOT NULL,
|
|---|
| 152 | TSTARTI mediumint UNSIGNED NOT NULL,
|
|---|
| 153 | TSTOPF double NOT NULL,
|
|---|
| 154 | TSTOPI mediumint UNSIGNED NOT NULL,
|
|---|
| 155 | DNA0 bigint UNSIGNED NOT NULL,
|
|---|
| 156 | DNA1 bigint UNSIGNED NOT NULL,
|
|---|
| 157 | FWVER0 mediumint UNSIGNED NOT NULL,
|
|---|
| 158 | FWVER1 mediumint UNSIGNED NOT NULL,
|
|---|
| 159 | PRIMARY KEY (Telescope, NIGHT, RUNID),
|
|---|
| 160 | KEY (RUNTYPE)
|
|---|
| 161 | );
|
|---|
| 162 | "\ > ${OUTPUT}
|
|---|
| 163 |
|
|---|
| 164 | if [ ${DELETE} = "true" ]
|
|---|
| 165 | then
|
|---|
| 166 | echo "DELETE FROM RawData" >> ${OUTPUT}
|
|---|
| 167 | echo "WHERE Telescope = ${TEL}" >> ${OUTPUT}
|
|---|
| 168 | echo "AND NIGHT = ${NIGHT};" >> ${OUTPUT}
|
|---|
| 169 | echo "" >> ${OUTPUT}
|
|---|
| 170 | fi
|
|---|
| 171 |
|
|---|
| 172 |
|
|---|
| 173 | # Loop over all input arguments
|
|---|
| 174 | for ROOT in "${SUBDIR}"/*
|
|---|
| 175 | do
|
|---|
| 176 |
|
|---|
| 177 | # check if file extension is correct
|
|---|
| 178 | if [ "${ROOT: -8}" == ".fits.fz" ]
|
|---|
| 179 | then
|
|---|
| 180 | echo "${ROOT}"
|
|---|
| 181 | RUNID=`basename "${ROOT}" .fits.fz | cut -f2 -d "_"`
|
|---|
| 182 | else
|
|---|
| 183 | echo "${ROOT} has wrong file extension and will not be processed."
|
|---|
| 184 | continue
|
|---|
| 185 | fi
|
|---|
| 186 | EXT=0
|
|---|
| 187 |
|
|---|
| 188 | # Extract the header from the fits file with fitsdump,
|
|---|
| 189 | # the following chain properly formats the output,
|
|---|
| 190 | # replaces T/F with true/false, replaces run-ytpes by numeric
|
|---|
| 191 | # types, adds enclosures for SQL variable names where
|
|---|
| 192 | # necessary, removes enclosures around hex-numbers and adds
|
|---|
| 193 | # commas after all lines except the last one
|
|---|
| 194 | RESULT=`${FITSDUMP} -h "${ROOT}" 2>&4 \
|
|---|
| 195 | | grep "${TOGREP}" \
|
|---|
| 196 | | grep -v CHECKSUM \
|
|---|
| 197 | | cut -c4- \
|
|---|
| 198 | | cut -f1 -d\/ \
|
|---|
| 199 | | sed "s/'drs-pedestal'/1/g" \
|
|---|
| 200 | | sed "s/'drs-gain'/2/g" \
|
|---|
| 201 | | sed "s/'pedestal'/3/g" \
|
|---|
| 202 | | sed "s/'data'/4/g" \
|
|---|
| 203 | | sed "s/'custom'/0/g" \
|
|---|
| 204 | | sed 's/\ T\ /true/g' \
|
|---|
| 205 | | sed 's/\ F\ /false/g' \
|
|---|
| 206 | | sed 's/^DATE-END/\`DATE-END\`/' \
|
|---|
| 207 | | sed 's/^DATE-OBS/\`DATE-OBS\`/' \
|
|---|
| 208 | | sed 's/^DATE[^-]/\`DATE\`/' \
|
|---|
| 209 | | sed "s/^\(.*\)'\(0x[0-9a-f]*\)'\(.*\)$/\1\2\3/g" \
|
|---|
| 210 | | sed '$!s/$/,/'` || EXT=1
|
|---|
| 211 |
|
|---|
| 212 | if [ $EXT -ne 0 ]
|
|---|
| 213 | then
|
|---|
| 214 | echo "${ROOT} Fitsdump failed!"
|
|---|
| 215 | SetRawStatus "${ROOT}" ${RUNID} "1"
|
|---|
| 216 | continue
|
|---|
| 217 | fi
|
|---|
| 218 |
|
|---|
| 219 | # compare to refference header and count commas
|
|---|
| 220 | CHECKSHORT=`echo ${RESULT} | grep -o "${SEARCHSHORT}"`
|
|---|
| 221 | CHECKLONG=`echo ${RESULT} | grep -o "${SEARCHLONG}"`
|
|---|
| 222 | CNTCHECK=`echo "${RESULT}" | grep -o ',' | wc -l`
|
|---|
| 223 |
|
|---|
| 224 | # If long or short CHECK are matching, write result to file
|
|---|
| 225 | if [ "${CHECKLONG}" == "${REFLONG}" ] &&
|
|---|
| 226 | [ "${CNTCHECK}" == "${CNTLONG}" ]
|
|---|
| 227 | then
|
|---|
| 228 | InsertRaw "${RESULT}"
|
|---|
| 229 | SetRawStatus "${ROOT}" ${RUNID} "0"
|
|---|
| 230 | elif [ "${CHECKSHORT}" == "${REFSHORT}" ] &&
|
|---|
| 231 | [ "${CNTCHECK}" == "${CNTSHORT}" ]
|
|---|
| 232 | then
|
|---|
| 233 | InsertRaw "${RESULT}"
|
|---|
| 234 | SetRawStatus "${ROOT}" ${RUNID} "0"
|
|---|
| 235 | else
|
|---|
| 236 | echo "${ROOT} header not valid!"
|
|---|
| 237 | SetRawStatus "${ROOT}" ${RUNID} "2"
|
|---|
| 238 | fi
|
|---|
| 239 |
|
|---|
| 240 | done
|
|---|
