| 1 | #!/bin/bash
|
|---|
| 2 | #
|
|---|
| 3 | # Synchronizes the HAWC's Eye data on disk with the SQL database.
|
|---|
| 4 | #
|
|---|
| 5 | # The following scripts must be in the same directory as this script and
|
|---|
| 6 | # the named variables must be eddited in those files.
|
|---|
| 7 | # run-scripts.sh (this script)
|
|---|
| 8 | # TEL
|
|---|
| 9 | # DIR
|
|---|
| 10 | # CRED
|
|---|
| 11 | # CALLISTORC
|
|---|
| 12 | # STARRC
|
|---|
| 13 | # update-file-table2.sh
|
|---|
| 14 | # extract-raw-header2.sh
|
|---|
| 15 | # FITSDUMP
|
|---|
| 16 | # write-calibration-query.sh
|
|---|
| 17 | # extract-aux-data2.sh
|
|---|
| 18 | # MARS
|
|---|
| 19 | # MACROS
|
|---|
| 20 | # run-callisto2.sh
|
|---|
| 21 | # MARS
|
|---|
| 22 | # MACROS
|
|---|
| 23 | # run-star2.sh
|
|---|
| 24 | # MARS
|
|---|
| 25 | # MACROS
|
|---|
| 26 | # run-root2sql2.sh
|
|---|
| 27 | # ROOT2SQL
|
|---|
| 28 | #
|
|---|
| 29 | # Example:
|
|---|
| 30 | # bash run-scripts.sh
|
|---|
| 31 |
|
|---|
| 32 | set -o errexit
|
|---|
| 33 | set -o errtrace
|
|---|
| 34 | set -o nounset
|
|---|
| 35 | set -o pipefail
|
|---|
| 36 |
|
|---|
| 37 |
|
|---|
| 38 |
|
|---|
| 39 | function LockExit()
|
|---|
| 40 | {
|
|---|
| 41 | echo "ERROR: One instance of ${SCRPTN} already running" >&2
|
|---|
| 42 | exit 1
|
|---|
| 43 | }
|
|---|
| 44 |
|
|---|
| 45 | function DescExit()
|
|---|
| 46 | {
|
|---|
| 47 | echo "ERROR: Filedescriptor 200 already taken" >&2
|
|---|
| 48 | exit 1
|
|---|
| 49 | }
|
|---|
| 50 |
|
|---|
| 51 | function ErrExit()
|
|---|
| 52 | {
|
|---|
| 53 | echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
|
|---|
| 54 | exit 1
|
|---|
| 55 | }
|
|---|
| 56 |
|
|---|
| 57 | function StrgCExit()
|
|---|
| 58 | {
|
|---|
| 59 | echo " "
|
|---|
| 60 | echo "$0 was forcefully terminated" >&2
|
|---|
| 61 | exit 1
|
|---|
| 62 | }
|
|---|
| 63 |
|
|---|
| 64 | trap ErrExit ERR
|
|---|
| 65 | trap StrgCExit INT
|
|---|
| 66 |
|
|---|
| 67 | # Telescope for which to process
|
|---|
| 68 | readonly TEL="${1}"
|
|---|
| 69 | if [ ${TEL} = "1" ]
|
|---|
| 70 | then
|
|---|
| 71 | echo "Processing for Telescope 1"
|
|---|
| 72 | elif [ ${TEL} = "2" ]
|
|---|
| 73 | then
|
|---|
| 74 | echo "Processing for Telescope 2"
|
|---|
| 75 | else
|
|---|
| 76 | echo "No valid Telescope selected!" >&2
|
|---|
| 77 | exit 1
|
|---|
| 78 | fi
|
|---|
| 79 |
|
|---|
| 80 | # Filename
|
|---|
| 81 | readonly SCRPTN=`basename $0 .sh`
|
|---|
| 82 |
|
|---|
| 83 | # For lockfile
|
|---|
| 84 | readonly LOCK="/tmp/${SCRPTN}${TEL}.lock"
|
|---|
| 85 |
|
|---|
| 86 | eval "exec 20${TEL}>${LOCK}" || DescExit
|
|---|
| 87 | flock -n "20${TEL}" || LockExit
|
|---|
| 88 |
|
|---|
| 89 | cd /home/hawc/Desktop/DiskToDB/
|
|---|
| 90 |
|
|---|
| 91 | # For logfile
|
|---|
| 92 | mkdir -p log
|
|---|
| 93 | readonly DATE=`date +%Y%m%d_%H%M`
|
|---|
| 94 | readonly LOG="./log/${SCRPTN}_${DATE}_${TEL}.log"
|
|---|
| 95 | exec 3>&1 1>>${LOG}
|
|---|
| 96 | exec 4>>${LOG}
|
|---|
| 97 | exec 5>&2 2>>${LOG}
|
|---|
| 98 | # exec 3>&1
|
|---|
| 99 | # exec 4>&1
|
|---|
| 100 |
|
|---|
| 101 | echo "Start processing data"
|
|---|
| 102 |
|
|---|
| 103 | function DeleteEntries()
|
|---|
| 104 | {
|
|---|
| 105 | # Delete entries form table ${3} for a given Telescope and Night
|
|---|
| 106 | # Arguments: Credentials Telescope Table Night
|
|---|
| 107 | local cred="${1}"
|
|---|
| 108 | local tel="${2}"
|
|---|
| 109 | local table="${3}"
|
|---|
| 110 | local night="${4}"
|
|---|
| 111 |
|
|---|
| 112 | mysql --defaults-file="${cred}" \
|
|---|
| 113 | --skip-column-names \
|
|---|
| 114 | --batch \
|
|---|
| 115 | --raw \
|
|---|
| 116 | --compress \
|
|---|
| 117 | -e "DELETE FROM ${table} WHERE Telescope = ${tel} \
|
|---|
| 118 | AND NIGHT = ${night};"
|
|---|
| 119 | }
|
|---|
| 120 |
|
|---|
| 121 | function SetStatus()
|
|---|
| 122 | {
|
|---|
| 123 | # Status update for one Column, Night and Telescope
|
|---|
| 124 | # Arguments: Credentials Telescope Column Value Night
|
|---|
| 125 | local cred="${1}"
|
|---|
| 126 | local tel="${2}"
|
|---|
| 127 | local col="${3}"
|
|---|
| 128 | local val="${4}"
|
|---|
| 129 | local night="${5}"
|
|---|
| 130 |
|
|---|
| 131 | # Check if a valid Column was selected
|
|---|
| 132 | if [ ${col} != "header" ] \
|
|---|
| 133 | && [ ${col} != "calibration" ] \
|
|---|
| 134 | && [ ${col} != "auxiliary" ] \
|
|---|
| 135 | && [ ${col} != "callisto" ] \
|
|---|
| 136 | && [ ${col} != "star" ]
|
|---|
| 137 | then
|
|---|
| 138 | echo "ERROR: ${col} is not a valid column."
|
|---|
| 139 | exit 1
|
|---|
| 140 | fi
|
|---|
| 141 |
|
|---|
| 142 | mysql --defaults-file="${cred}" \
|
|---|
| 143 | --skip-column-names \
|
|---|
| 144 | --batch \
|
|---|
| 145 | --raw \
|
|---|
| 146 | --compress \
|
|---|
| 147 | -e "UPDATE DataOnDisk SET ${col} = ${val}\
|
|---|
| 148 | WHERE NIGHT = ${night} AND Telescope = ${tel};"
|
|---|
| 149 | }
|
|---|
| 150 |
|
|---|
| 151 | function GetStatus()
|
|---|
| 152 | {
|
|---|
| 153 | # Return status of Night for a given Column and Telescope
|
|---|
| 154 | # Arguments: Credentials Telescope Column Night
|
|---|
| 155 | local cred="${1}"
|
|---|
| 156 | local tel="${2}"
|
|---|
| 157 | local col="${3}"
|
|---|
| 158 | local night="${4}"
|
|---|
| 159 |
|
|---|
| 160 | # Check if a valid Column was selected
|
|---|
| 161 | if [ ${col} != "header" ] \
|
|---|
| 162 | && [ ${col} != "calibration" ] \
|
|---|
| 163 | && [ ${col} != "auxiliary" ] \
|
|---|
| 164 | && [ ${col} != "callisto" ] \
|
|---|
| 165 | && [ ${col} != "star" ]
|
|---|
| 166 | then
|
|---|
| 167 | echo "ERROR: ${col} is not a valid column."
|
|---|
| 168 | exit 1
|
|---|
| 169 | fi
|
|---|
| 170 |
|
|---|
| 171 | local STATUS=`mysql --defaults-file="${cred}" \
|
|---|
| 172 | --skip-column-names \
|
|---|
| 173 | --batch \
|
|---|
| 174 | --raw \
|
|---|
| 175 | --compress \
|
|---|
| 176 | -e "SELECT DISTINCT ${col} FROM DataOnDisk \
|
|---|
| 177 | WHERE NIGHT = ${night} \
|
|---|
| 178 | AND Telescope = ${tel} \
|
|---|
| 179 | AND ${col} IS NULL;"`
|
|---|
| 180 |
|
|---|
| 181 | echo "${STATUS}"
|
|---|
| 182 | }
|
|---|
| 183 |
|
|---|
| 184 |
|
|---|
| 185 | function DoesDirExists()
|
|---|
| 186 | {
|
|---|
| 187 | local thing="${1}"
|
|---|
| 188 | # Check if thing exists
|
|---|
| 189 | if [ ! -d "${thing}" ]
|
|---|
| 190 | then
|
|---|
| 191 | echo "${thing} is not a valid Path. Please change variable in script ${SCRPTN}."
|
|---|
| 192 | exit 1
|
|---|
| 193 | fi
|
|---|
| 194 | }
|
|---|
| 195 |
|
|---|
| 196 | function DoesFileExists()
|
|---|
| 197 | {
|
|---|
| 198 | local thing="${1}"
|
|---|
| 199 | # Check if thing exists
|
|---|
| 200 | if [ ! -f "${thing}" ]
|
|---|
| 201 | then
|
|---|
| 202 | echo "${thing} is not a valid File. Please change variable in script ${SCRPTN}."
|
|---|
| 203 | exit 1
|
|---|
| 204 | fi
|
|---|
| 205 | }
|
|---|
| 206 |
|
|---|
| 207 | # Define variables
|
|---|
| 208 | # Directory in which the raw directory is located
|
|---|
| 209 | readonly DIR="/media/DATA/HAWCsEYE0${TEL}/"
|
|---|
| 210 | # Credentials for the SQL database
|
|---|
| 211 | readonly CRED='../Credentials/credentials-read-only.cnf'
|
|---|
| 212 | # Credentials for root2sql for callisto
|
|---|
| 213 | readonly CALLISTORC='../Credentials/root2sql-callisto.rc'
|
|---|
| 214 | # Credentials for root2sql for star
|
|---|
| 215 | readonly STARRC='../Credentials/root2sql-star.rc'
|
|---|
| 216 | # ROOT init path
|
|---|
| 217 | readonly RPATH='../root-6.24.02/bin/thisroot.sh'
|
|---|
| 218 |
|
|---|
| 219 | echo "Processing data from ${DIR}"
|
|---|
| 220 |
|
|---|
| 221 | # Check if variable paths point to something sensible
|
|---|
| 222 | DoesDirExists "${DIR}"
|
|---|
| 223 | DoesFileExists "${CRED}"
|
|---|
| 224 | DoesFileExists "${CALLISTORC}"
|
|---|
| 225 | DoesFileExists "${STARRC}"
|
|---|
| 226 | DoesFileExists "${RPATH}"
|
|---|
| 227 |
|
|---|
| 228 | # SQL outputs
|
|---|
| 229 | mkdir -p ./queries
|
|---|
| 230 | readonly ONDISK="./queries/insert-OnDisk-${TEL}.sql"
|
|---|
| 231 | readonly AUXDISK="./queries/insert-AuxOnDisk-${TEL}.sql"
|
|---|
| 232 | readonly RAW="./queries/insert-raw-${TEL}.sql"
|
|---|
| 233 | readonly CALIB="./queries/find-calibration2-${TEL}.sql"
|
|---|
| 234 | readonly AUX="./queries/insert-aux-${TEL}.sql"
|
|---|
| 235 |
|
|---|
| 236 | # Init ROOT
|
|---|
| 237 | set +o nounset
|
|---|
| 238 | . "${RPATH}"
|
|---|
| 239 | set -o nounset
|
|---|
| 240 |
|
|---|
| 241 | # Create Database of existing files
|
|---|
| 242 | echo "Updating file table"
|
|---|
| 243 | bash ./update-file-table2.sh -i -o "${ONDISK}" "${TEL}" "${DIR}/raw"
|
|---|
| 244 | # Update DataOnDisk tabel in database
|
|---|
| 245 | echo "Uploading updated table"
|
|---|
| 246 | mysql --defaults-file="${CRED}" < "${ONDISK}"
|
|---|
| 247 |
|
|---|
| 248 | # Create Database of existing auxiliary files
|
|---|
| 249 | bash ./update-aux-table.sh -r -o "${AUXDISK}" "${TEL}" "${DIR}/auxil"
|
|---|
| 250 | # Update DataOnDisk tabel in database
|
|---|
| 251 | echo "Uploading updated table"
|
|---|
| 252 | mysql --defaults-file="${CRED}" < "${AUXDISK}"
|
|---|
| 253 |
|
|---|
| 254 | # get list of NIGHTS with unprocessed files
|
|---|
| 255 | echo "Getting nights to be processed"
|
|---|
| 256 | NIGHTS=`mysql --defaults-file="${CRED}" \
|
|---|
| 257 | --skip-column-names \
|
|---|
| 258 | --batch \
|
|---|
| 259 | --raw \
|
|---|
| 260 | --compress \
|
|---|
| 261 | -e "SELECT DISTINCT NIGHT FROM DataOnDisk \
|
|---|
| 262 | WHERE header IS NULL \
|
|---|
| 263 | OR calibration IS NULL \
|
|---|
| 264 | OR auxiliary IS NULL \
|
|---|
| 265 | OR callisto IS NULL \
|
|---|
| 266 | OR star IS NULL \
|
|---|
| 267 | AND Telescope = ${TEL};"`
|
|---|
| 268 |
|
|---|
| 269 | # Check if NIGHTS is empty and dont do anythin if so.
|
|---|
| 270 | if [ -z "${NIGHTS}" ]
|
|---|
| 271 | then
|
|---|
| 272 | echo "Database is up to date. Nothing to do here."
|
|---|
| 273 | exit 0
|
|---|
| 274 | fi
|
|---|
| 275 |
|
|---|
| 276 | echo "Updating database"
|
|---|
| 277 |
|
|---|
| 278 | echo "Nights to process:"
|
|---|
| 279 | echo "${NIGHTS}"
|
|---|
| 280 | echo " "
|
|---|
| 281 |
|
|---|
| 282 |
|
|---|
| 283 | # Execute scripts on all RunId for given Telescope and Night that has
|
|---|
| 284 | # Loop over nights and upload to database for every loop
|
|---|
| 285 | for NIGHT in ${NIGHTS}
|
|---|
| 286 | do
|
|---|
| 287 | HEAD_S=`GetStatus ${CRED} ${TEL} "header" ${NIGHT}`
|
|---|
| 288 | CALI_S=`GetStatus ${CRED} ${TEL} "calibration" ${NIGHT}`
|
|---|
| 289 | AUXI_S=`GetStatus ${CRED} ${TEL} "auxiliary" ${NIGHT}`
|
|---|
| 290 | CALL_S=`GetStatus ${CRED} ${TEL} "callisto" ${NIGHT}`
|
|---|
| 291 | STAR_S=`GetStatus ${CRED} ${TEL} "star" ${NIGHT}`
|
|---|
| 292 |
|
|---|
| 293 | echo "header = ${HEAD_S} for night = ${NIGHT}"
|
|---|
| 294 | echo " "
|
|---|
| 295 |
|
|---|
| 296 | # If header is null
|
|---|
| 297 | if [ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]
|
|---|
| 298 | then
|
|---|
| 299 | # Get raw data headers
|
|---|
| 300 | echo "== In Status 1 =="
|
|---|
| 301 | bash ./extract-raw-header2.sh -d -o "${RAW}" \
|
|---|
| 302 | ${TEL} \
|
|---|
| 303 | "${DIR}"/raw \
|
|---|
| 304 | ${NIGHT}
|
|---|
| 305 | echo "Uploading raw headers for night: ${NIGHT}"
|
|---|
| 306 | mysql --defaults-file="${CRED}" < "${RAW}"
|
|---|
| 307 | echo "Finished uploading raw headers for night: ${NIGHT}"
|
|---|
| 308 | echo " "
|
|---|
| 309 | fi
|
|---|
| 310 |
|
|---|
| 311 | echo "calibration = ${CALI_S} for night = ${NIGHT}"
|
|---|
| 312 | echo " "
|
|---|
| 313 |
|
|---|
| 314 | # If header is null and or calibration is null
|
|---|
| 315 | if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
|
|---|
| 316 | || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ])
|
|---|
| 317 | then
|
|---|
| 318 | # Get calibration files
|
|---|
| 319 | echo "== In Status 2 =="
|
|---|
| 320 | bash ./write-calibration-query.sh -d -o "${CALIB}" \
|
|---|
| 321 | ${TEL} \
|
|---|
| 322 | ${NIGHT}
|
|---|
| 323 | echo "Uploading calibration for night: ${NIGHT}"
|
|---|
| 324 | mysql --defaults-file="${CRED}" < "${CALIB}"
|
|---|
| 325 | echo "Finished uploading calibration for night: ${NIGHT}"
|
|---|
| 326 | echo " "
|
|---|
| 327 | fi
|
|---|
| 328 |
|
|---|
| 329 | echo "auxiliary = ${AUXI_S} for night = ${NIGHT}"
|
|---|
| 330 | echo " "
|
|---|
| 331 |
|
|---|
| 332 | # If header is null and or calibration is null and or auxiliary is null
|
|---|
| 333 | if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
|
|---|
| 334 | || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
|
|---|
| 335 | || ([ ! -z ${AUXI_S} ] && [ ${AUXI_S} = "NULL" ])
|
|---|
| 336 | then
|
|---|
| 337 | # Get aux data
|
|---|
| 338 | echo "== In Status 3 =="
|
|---|
| 339 | bash ./extract-aux-data2.sh -d -o "${AUX}" \
|
|---|
| 340 | ${CRED} \
|
|---|
| 341 | ${TEL} \
|
|---|
| 342 | "${DIR}"/auxil \
|
|---|
| 343 | ${NIGHT}
|
|---|
| 344 | echo "Uploading aux data for night: ${NIGHT}"
|
|---|
| 345 | mysql --defaults-file="${CRED}" < "${AUX}"
|
|---|
| 346 | echo "Finished uploading aux data for night: ${NIGHT}"
|
|---|
| 347 | echo " "
|
|---|
| 348 | fi
|
|---|
| 349 |
|
|---|
| 350 | echo "callisto = ${CALL_S} for night = ${NIGHT}"
|
|---|
| 351 | echo " "
|
|---|
| 352 |
|
|---|
| 353 | # If header is null and or calibration is null and or callisto is null
|
|---|
| 354 | if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
|
|---|
| 355 | || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
|
|---|
| 356 | || ([ ! -z ${CALL_S} ] && [ ${CALL_S} = "NULL" ])
|
|---|
| 357 | then
|
|---|
| 358 | # Get callisto data
|
|---|
| 359 | echo "== In Status 4 =="
|
|---|
| 360 | bash ./run-callisto2.sh -r \
|
|---|
| 361 | "${CRED}" \
|
|---|
| 362 | ${TEL} \
|
|---|
| 363 | "${DIR}"/raw \
|
|---|
| 364 | ${NIGHT}
|
|---|
| 365 |
|
|---|
| 366 | echo "Deleting callisto entries for night: ${NIGHT}"
|
|---|
| 367 | DeleteEntries ${CRED} ${TEL} "Callisto" ${NIGHT}
|
|---|
| 368 |
|
|---|
| 369 | echo "Uploading callisto entries for night: ${NIGHT}"
|
|---|
| 370 | bash ./run-root2sql2.sh -Y \
|
|---|
| 371 | "${CRED}" \
|
|---|
| 372 | "${CALLISTORC}" \
|
|---|
| 373 | ${TEL} \
|
|---|
| 374 | "${DIR}"/callisto \
|
|---|
| 375 | ${NIGHT}
|
|---|
| 376 | echo "Finished uploading callisto entries for night: ${NIGHT}"
|
|---|
| 377 | echo " "
|
|---|
| 378 | fi
|
|---|
| 379 |
|
|---|
| 380 | echo "star = ${STAR_S} for night = ${NIGHT}"
|
|---|
| 381 | echo " "
|
|---|
| 382 |
|
|---|
| 383 | # If header is null and or calibration is null and or callisto is null
|
|---|
| 384 | # and or star is null
|
|---|
| 385 | if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
|
|---|
| 386 | || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
|
|---|
| 387 | || ([ ! -z ${CALL_S} ] && [ ${CALL_S} = "NULL" ]) \
|
|---|
| 388 | || ([ ! -z ${STAR_S} ] && [ ${STAR_S} = "NULL" ])
|
|---|
| 389 | then
|
|---|
| 390 | # Get star data
|
|---|
| 391 | echo "== In Status 5 =="
|
|---|
| 392 | bash ./run-star2.sh -r "${CRED}" ${TEL} "${DIR}"/raw ${NIGHT}
|
|---|
| 393 |
|
|---|
| 394 | echo "Deleting star entries for night: ${NIGHT}"
|
|---|
| 395 | DeleteEntries ${CRED} ${TEL} "Star" ${NIGHT}
|
|---|
| 396 |
|
|---|
| 397 | echo "Uploading star entries for night: ${NIGHT}"
|
|---|
| 398 | bash ./run-root2sql2.sh -I \
|
|---|
| 399 | "${CRED}" \
|
|---|
| 400 | "${STARRC}" \
|
|---|
| 401 | ${TEL} \
|
|---|
| 402 | "${DIR}"/star \
|
|---|
| 403 | ${NIGHT}
|
|---|
| 404 | echo "Finished uploading star entries for night: ${NIGHT}"
|
|---|
| 405 | echo " "
|
|---|
| 406 | fi
|
|---|
| 407 |
|
|---|
| 408 | done
|
|---|
| 409 |
|
|---|
| 410 | echo " "
|
|---|
| 411 | echo "Finished processing data"
|
|---|
