source: trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh@ 20103

Last change on this file since 20103 was 20103, checked in by maslowski, 3 years ago
Added better error handeling and tracking in the databas.
File size: 8.8 KB
Line 
1#!/bin/bash
2#
3# Synchronizes the HAWC's Eye data on disk with the SQL database.
4#
5# The following scripts must be in the same directory as this script and
6# the named variables must be eddited in those files.
7# run-scripts.sh (this script)
8# TEL
9# DIR
10# CRED
11# CALLISTORC
12# STARRC
13# update-file-table2.sh
14# extract-raw-header2.sh
15# FITSDUMP
16# write-calibration-query.sh
17# extract-aux-data2.sh
18# MARS
19# MACROS
20# run-callisto2.sh
21# MARS
22# MACROS
23# run-star2.sh
24# MARS
25# MACROS
26# run-root2sql2.sh
27# ROOT2SQL
28#
29# Example:
30# bash run-scripts.sh
31
32set -o errexit
33set -o errtrace
34set -o nounset
35set -o pipefail
36
37
38# Filename
39readonly SCRPTN=`basename $0 .sh`
40
41# For lockfile
42readonly LOCK="/tmp/${SCRPTN}.lock"
43
44function LockExit()
45{
46 echo "ERROR: One instance of ${SCRPTN} already running" >&2
47 exit 1
48}
49
50function DescExit()
51{
52 echo "ERROR: Filedescriptor 200 already taken" >&2
53 exit 1
54}
55
56function ErrExit()
57{
58 echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
59 exit 1
60}
61
62function StrgCExit()
63{
64 echo " "
65 echo "$0 was forcefully terminated" >&2
66 exit 1
67}
68
69trap ErrExit ERR
70trap StrgCExit INT
71
72exec 200>${LOCK} || DescExit
73flock -n 200 || LockExit
74
75# For logfile
76mkdir -p log
77readonly DATE=`date +%Y%m%d_%H%M`
78readonly LOG="./log/${SCRPTN}_${DATE}.log"
79# exec 3>&1 1>>${LOG}
80# exec 4>>${LOG}
81exec 3>&1
82exec 4>&1
83
84echo "Start processing data"
85
86function DeleteEntries()
87{
88 # Delete entries form table ${3} for a given Telescope and Night
89 # Arguments: Credentials Telescope Table Night
90 local cred="${1}"
91 local tel="${2}"
92 local table="${3}"
93 local night="${4}"
94
95 mysql --defaults-file="${cred}" \
96 --skip-column-names \
97 --batch \
98 --raw \
99 --compress \
100 -e "DELETE FROM ${table} WHERE Telescope = ${tel} \
101 AND NIGHT = ${night};"
102}
103
104function SetStatus()
105{
106 # Status update for one Column, Night and Telescope
107 # Arguments: Credentials Telescope Column Value Night
108 local cred="${1}"
109 local tel="${2}"
110 local col="${3}"
111 local val="${4}"
112 local night="${5}"
113
114 # Check if a valid Column was selected
115 if [ ${col} != "header" ] \
116 && [ ${col} != "calibration" ] \
117 && [ ${col} != "auxiliary" ] \
118 && [ ${col} != "callisto" ] \
119 && [ ${col} != "star" ]
120 then
121 echo "ERROR: ${col} is not a valid column."
122 exit 1
123 fi
124
125 mysql --defaults-file="${cred}" \
126 --skip-column-names \
127 --batch \
128 --raw \
129 --compress \
130 -e "UPDATE DataOnDisk SET ${col} = ${val}\
131 WHERE NIGHT = ${night} AND Telescope = ${tel};"
132}
133
134function GetStatus()
135{
136 # Return status of Night for a given Column and Telescope
137 # Arguments: Credentials Telescope Column Night
138 local cred="${1}"
139 local tel="${2}"
140 local col="${3}"
141 local night="${4}"
142
143 # Check if a valid Column was selected
144 if [ ${col} != "header" ] \
145 && [ ${col} != "calibration" ] \
146 && [ ${col} != "auxiliary" ] \
147 && [ ${col} != "callisto" ] \
148 && [ ${col} != "star" ]
149 then
150 echo "ERROR: ${col} is not a valid column."
151 exit 1
152 fi
153
154 local STATUS=`mysql --defaults-file="${cred}" \
155 --skip-column-names \
156 --batch \
157 --raw \
158 --compress \
159 -e "SELECT DISTINCT ${col} FROM DataOnDisk \
160 WHERE NIGHT = ${night} \
161 AND Telescope = ${tel} \
162 AND ${col} IS NULL;"`
163
164 echo "${STATUS}"
165}
166
167
168function DoesDirExists()
169{
170 local thing="${1}"
171 # Check if thing exists
172 if [ ! -d "${thing}" ]
173 then
174 echo "${thing} is not a valid Path. Please change variable in script ${SCRPTN}."
175 exit 1
176 fi
177}
178
179function DoesFileExists()
180{
181 local thing="${1}"
182 # Check if thing exists
183 if [ ! -f "${thing}" ]
184 then
185 echo "${thing} is not a valid File. Please change variable in script ${SCRPTN}."
186 exit 1
187 fi
188}
189
190
191# Define variables
192# Telescope for which to process
193readonly TEL=1
194# Directory in which the raw data is located
195readonly DIR='/run/media/frankm/Neuer Datenträger/data/DATA/HE0'${TEL}
196# Credentials for the SQL database
197readonly CRED='../credentials-read-only.cnf'
198# Credentials for root2sql for callisto
199readonly CALLISTORC='../root2sql-callisto.rc'
200# Credentials for root2sql for star
201readonly STARRC='../root2sql-star.rc'
202
203# Check if variable paths point to something sensible
204DoesDirExists "${DIR}"
205DoesFileExists "${CRED}"
206DoesFileExists "${CALLISTORC}"
207DoesFileExists "${STARRC}"
208
209# SQL outputs
210mkdir -p ./queries
211readonly ONDISK='./queries/insert-OnDisk.sql'
212readonly RAW='./queries/insert-raw.sql'
213readonly CALIB='./queries/find-calibration2.sql'
214readonly AUX='./queries/insert-aux.sql'
215
216# Create Database of existing files
217# Create SQL query with:
218# (Telescope, Night, RUNID, ISDRSFILE).
219echo "Updating file table"
220bash ./update-file-table2.sh -i -o "${ONDISK}" "${TEL}" "${DIR}/raw"
221# Update DataOnDisk tabel in database
222echo "Uploading updated table"
223mysql --defaults-file="${CRED}" < "${ONDISK}"
224
225# get list of NIGHTS with unprocessed files
226echo "Getting nights to be processed"
227NIGHTS=`mysql --defaults-file="${CRED}" \
228 --skip-column-names \
229 --batch \
230 --raw \
231 --compress \
232 -e "SELECT DISTINCT NIGHT FROM DataOnDisk \
233 WHERE header IS NULL \
234 OR calibration IS NULL \
235 OR auxiliary IS NULL \
236 OR callisto IS NULL \
237 OR star IS NULL \
238 AND Telescope = ${TEL};"`
239
240# Check if NIGHTS is empty and dont do anythin if so.
241if [ -z "${NIGHTS}" ]
242then
243 echo "Database is up to date. Nothing to do here."
244 exit 0
245fi
246
247echo "Updating database"
248
249NIGHTS=20201111
250
251echo "Nights to process:"
252echo "${NIGHTS}"
253echo " "
254
255
256# Execute scripts on all RunId for given Telescope and Night that has
257# Loop over nights and upload to database for every loop
258for NIGHT in ${NIGHTS}
259do
260 HEAD_S=`GetStatus ${CRED} ${TEL} "header" ${NIGHT}`
261 CALI_S=`GetStatus ${CRED} ${TEL} "calibration" ${NIGHT}`
262 AUXI_S=`GetStatus ${CRED} ${TEL} "auxiliary" ${NIGHT}`
263 CALL_S=`GetStatus ${CRED} ${TEL} "callisto" ${NIGHT}`
264 STAR_S=`GetStatus ${CRED} ${TEL} "star" ${NIGHT}`
265
266 echo "header = ${HEAD_S} for night = ${NIGHT}"
267 echo " "
268
269 # If header is null
270 if [ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]
271 then
272 # Get raw data headers
273 echo "== In Status 1 =="
274 bash ./extract-raw-header2.sh -d -o "${RAW}" \
275 ${TEL} \
276 "${DIR}"/raw \
277 ${NIGHT}
278 echo "Uploading raw headers for night: ${NIGHT}"
279 mysql --defaults-file="${CRED}" < "${RAW}"
280 echo "Finished uploading raw headers for night: ${NIGHT}"
281 echo " "
282 fi
283
284 echo "calibration = ${CALI_S} for night = ${NIGHT}"
285 echo " "
286
287 # If header is null and or calibration is null
288 if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
289 || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ])
290 then
291 # Get calibration files
292 echo "== In Status 2 =="
293 bash ./write-calibration-query.sh -d -o "${CALIB}" \
294 ${TEL} \
295 ${NIGHT}
296 echo "Uploading calibration for night: ${NIGHT}"
297 mysql --defaults-file="${CRED}" < "${CALIB}"
298 echo "Finished uploading calibration for night: ${NIGHT}"
299 echo " "
300 fi
301
302 echo "auxiliary = ${AUXI_S} for night = ${NIGHT}"
303 echo " "
304
305 # If header is null and or calibration is null and or auxiliary is null
306 if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
307 || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
308 || ([ ! -z ${AUXI_S} ] && [ ${AUXI_S} = "NULL" ])
309 then
310 # Get aux data
311 echo "== In Status 3 =="
312 bash ./extract-aux-data2.sh -d -o "${AUX}" \
313 ${CRED} \
314 ${TEL} \
315 "${DIR}"/auxil \
316 ${NIGHT}
317 echo "Uploading aux data for night: ${NIGHT}"
318 mysql --defaults-file="${CRED}" < "${AUX}"
319 echo "Finished uploading aux data for night: ${NIGHT}"
320 echo " "
321 fi
322
323 echo "callisto = ${CALL_S} for night = ${NIGHT}"
324 echo " "
325
326 # If header is null and or calibration is null and or callisto is null
327 if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
328 || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
329 || ([ ! -z ${CALL_S} ] && [ ${CALL_S} = "NULL" ])
330 then
331 # Get callisto data
332 echo "== In Status 4 =="
333 bash ./run-callisto2.sh -r \
334 "${CRED}" \
335 ${TEL} \
336 "${DIR}"/raw \
337 ${NIGHT}
338
339 echo "Deleting callisto entries for night: ${NIGHT}"
340 DeleteEntries ${CRED} ${TEL} "Callisto" ${NIGHT}
341
342 echo "Uploading callisto entries for night: ${NIGHT}"
343 bash ./run-root2sql2.sh -Y \
344 "${CRED}" \
345 "${CALLISTORC}" \
346 ${TEL} \
347 "${DIR}"/callisto \
348 ${NIGHT}
349 echo "Finished uploading callisto entries for night: ${NIGHT}"
350 echo " "
351 fi
352
353 echo "star = ${STAR_S} for night = ${NIGHT}"
354 echo " "
355
356 # If header is null and or calibration is null and or callisto is null
357 # and or star is null
358 if ([ ! -z ${HEAD_S} ] && [ ${HEAD_S} = "NULL" ]) \
359 || ([ ! -z ${CALI_S} ] && [ ${CALI_S} = "NULL" ]) \
360 || ([ ! -z ${CALL_S} ] && [ ${CALL_S} = "NULL" ]) \
361 || ([ ! -z ${STAR_S} ] && [ ${STAR_S} = "NULL" ])
362 then
363 # Get star data
364 echo "== In Status 5 =="
365 bash ./run-star2.sh -r "${CRED}" ${TEL} "${DIR}"/raw ${NIGHT}
366
367 echo "Deleting star entries for night: ${NIGHT}"
368 DeleteEntries ${CRED} ${TEL} "Star" ${NIGHT}
369
370 echo "Uploading star entries for night: ${NIGHT}"
371 bash ./run-root2sql2.sh -I \
372 "${CRED}" \
373 "${STARRC}" \
374 ${TEL} \
375 "${DIR}"/star \
376 ${NIGHT}
377 echo "Finished uploading star entries for night: ${NIGHT}"
378 echo " "
379 fi
380
381done
382
383echo " "
384echo "Finished processing data"
Note: See TracBrowser for help on using the repository browser.