source: trunk/DataCheck/Processing/FillAuxData.sh@ 18092

Last change on this file since 18092 was 18083, checked in by Daniela Dorner, 10 years ago
updated to let the scripts run on newdaq
  • Property svn:executable set to *
File size: 14.6 KB
Line 
1#!/bin/bash
2
3# new version of the script to run on newdaq
4
5# option whether to fill all row or only those where information is missing
6# $doupdate might be given as environment variable
7if [ "$doupdate" = "" ]
8then
9 doupdate="yes" # update all entries (needed when new fields have been added)
10 doupdate="no" # fill only entries which are not yet existing (default)
11fi
12
13source `dirname $0`/../Sourcefile.sh
14printprocesslog "INFO starting $0 with option doupdate="$doupdate
15
16logfile=$runlogpath"/FillAuxData-"$datetime".log"
17date >> $logfile
18
19# check if software is available
20if ! ls $factpath/fitsdump >/dev/null 2>&1
21then
22 printprocesslog "ERROR "$factpath"/fitsdump is not available."
23 finish
24fi
25
26# get dates
27if [ "$certaindate" != "" ]
28then
29 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
30 if [ "$checkstring" = "" ]
31 then
32 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
33 finish
34 fi
35 getdates $certaindate
36else
37 # get all night
38 #getdates "all"
39 # get last 3 nights if hour between 7 and 19h, else only current night
40 getdates 3 7 19
41fi
42
43
44printprocesslog "INFO processing the following night(s): "${dates[@]}
45echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
46
47# mjd of 1970-01-01
48# needed in this script as for 1 day the mjd in the aux files are inconsistent
49# mjdref=40587
50
51# known:
52# 2011/11/22 MJDREF in DRIVE empty, Time > 55000
53# 2011/11/23 MJDREF in DRIVE not empty, Time > 55000
54# 2011/11/24 MJDREF in DRIVE not empty, Time > 15000
55# raw files
56# 2011/11/21 no MJDREF
57# 2011/11/22 MJDREF
58# further things: https://www.fact-project.org/logbook/showthread.php?tid=67
59
60# trigger rate has as first value -1, but with using the median it should be fine
61
62
63function evaluatestatistics()
64{
65 # $1 variable name
66 # $@ statistics
67 if [ "$2" = "" ]
68 then
69 printprocesslog "WARN couldn't get statistics from file $1 for run "$date" "$file
70 continue
71 fi
72 min=
73 mean=
74 med=
75 max=
76 rms=
77 evaluation=`echo $@ | grep -E -o '\['${1}':0[:]?[0-9]*\]\ Min:\ [-]?[0-9]+[.]?[0-9]*\ Max:\ [-]?[0-9]+[.]?[0-9]*\ Med:\ [-]?[0-9]+[.]?[0-9]*\ Avg:\ [-]?[0-9]+[.]?[0-9]*\ Rms:\ [-]?[0-9]+[.]?[0-9]*[e]?[-]?[0-9]*'`
78 if [ "$evaluation" = "" ]
79 then
80 printprocesslog "WARN empty evaluation of statistic ("$@") for run "$date" "$file
81 #echo " ---> "$@
82 fi
83 #echo "ev: "$evaluation
84 min=`echo $evaluation | grep -E -o 'Min:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Min:\ //'`
85 max=`echo $evaluation | grep -E -o 'Max:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Max:\ //'`
86 med=`echo $evaluation | grep -E -o 'Med:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Med:\ //'`
87 mean=`echo $evaluation | grep -E -o 'Avg:\ [-]?[0-9]+[.]?[0-9]*' | sed -e 's/Avg:\ //'`
88 rms=`echo $evaluation | grep -E -o 'Rms:\ [-]?[0-9]+[.]?[0-9]*[e]?[-]?[0-9]+' | sed -e 's/Rms:\ //'`
89 #echo "eval: "$min" "$max" "$med" "$mean" "$rms
90}
91
92# do filling of aux data
93for date in ${dates[@]}
94do
95 auxdir=$auxdata/$date
96 rawdir=$rawdata/$date
97 runnumber=`echo $date | sed -e 's/\///g'`
98
99 # check if aux files are available from that night
100 if ! [ -d $auxdir ]
101 then
102 printprocesslog "INFO no data available in "$auxdir
103 continue
104 else
105 printprocesslog "INFO processing files in "$auxdir
106 #echo "INFO processing files in "$auxdir
107 fi
108
109 # check if raw files are available from that night
110 # only needed to get start/stop time
111 # might be removed once the start/stop time comes from the DB
112 if ! [ -d $rawdir ]
113 then
114 printprocesslog "INFO no data available in "$rawdir" -> continue"
115 continue
116 fi
117
118 # get file numbers from DB
119 # but only for not-corrupted files
120 # as aux files are written only once a minute, select only files which are older than 1.5 minutes
121 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 AND NOT ISNULL(fRunStop) AND fRunStop < SUBTIME(UTC_TIMESTAMP(), \"00:01:30\")"
122 # only runs which are not yet filled
123 if [ "$doupdate" = "no" ]
124 then
125 query=$query" AND ISNULL(fRightAscension) "
126 fi
127 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
128 filenumbers=( `sendquery $query` )
129 if [ ${#filenumbers} -eq 0 ]
130 then
131 printprocesslog "INFO No files found in the DB for night "$date
132 continue
133 fi
134
135 # get daily fits files
136 trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
137 if ! [ -e $trackingfile ]
138 then
139 printprocesslog "WARN "$trackingfile" not found."
140 fi
141
142 sourceposfile=$auxdir/$runnumber.DRIVE_CONTROL_SOURCE_POSITION.fits
143 if ! [ -e $sourceposfile ]
144 then
145 printprocesslog "WARN "$sourceposfile" not found."
146 else
147 sourceposfiletstarti=`$factpath/fitsdump -h $sourceposfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
148 sourceposfiletstartf=`$factpath/fitsdump -h $sourceposfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
149 if [ $sourceposfiletstarti -gt 30000 ]
150 then
151 sourceposfiletstart=`echo " $sourceposfiletstarti + $sourceposfiletstartf - 40587 " | bc -l`
152 else
153 sourceposfiletstart=`echo " $sourceposfiletstarti + $sourceposfiletstartf " | bc -l`
154 fi
155 fi
156
157 triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
158 if ! [ -e $triggerratefile ]
159 then
160 printprocesslog "WARN "$triggerratefile" not found."
161 fi
162
163 thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
164 if ! [ -e $thresholdfile ]
165 then
166 printprocesslog "WARN "$thresholdfile" not found."
167 fi
168
169 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
170 if ! [ -e $biasvoltagefile ]
171 then
172 printprocesslog "WARN "$biasvoltagefile" not found."
173 fi
174
175 # fill auxiliary information for files
176 for filenum in ${filenumbers[@]}
177 do
178 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
179 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
180 #echo `date`": processing file number "$runnumber"_"`printf %03d $filenum`
181
182 # get information from rawfile
183 rawfile=`ls $rawdir/$runnumber"_"\`printf %03d $filenum\`.fits*`
184 if ! [ -e $rawfile ]
185 then
186 printprocesslog "ERROR: "$rawfile" not found."
187 continue
188 fi
189
190 #runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
191 #mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
192 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
193 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
194 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
195 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
196 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
197 then
198 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
199 continue
200 fi
201 # assuming that at least TSTARTI and TSTOPI are consistent
202 #echo $rawfile
203 #echo $tstarti
204 #echo $tstopi
205 #echo $tstartf
206 #echo $tstopf
207 if [ $tstarti -gt 30000 ]
208 then
209 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
210 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
211 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
212 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
213 else
214 tstart=`echo " $tstarti + $tstartf " | bc -l`
215 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
216 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
217 tstop=`echo " $tstopi + $tstopf " | bc -l`
218 fi
219 #echo $tstart
220 #echo $tstop
221 #if [ $runnumber -eq 20111123 ]
222 #then
223 # # add mjdref for days were aux files were inconsistent
224 # tstart=`echo " $tstart + $mjdref " | bc -l`
225 # tstart2=`echo " $tstart2 + $mjdref " | bc -l`
226 # tstop=`echo " $tstop + $mjdref " | bc -l`
227 #fi
228
229 # get information from source_pos file
230 if [ -e $sourceposfile ]
231 then
232 sourcename=`$factpath/fitsdump ${sourceposfile} -c Time -c Name --filter='[1]<'${tstop}' && [1]>'${sourceposfiletstart} 2>/dev/null | tail -1 2>&1 | grep -o -E "['][a-zA-Z0-9\ \.\+\-]+[']" | sed -e "s/'//g"`
233 if [ "$sourcename" == "" ]
234 then
235 printprocesslog "INFO couldn't get sourcename ("$sourcename") from "$sourceposfile" for "$runnumber"_"$filenum
236 else
237 query="SELECT fSourceKey FROM Source WHERE fSourceName='"$sourcename"'"
238 sourcekey=`sendquery`
239 if [ "$sourcename" == "" ]
240 then
241 printprocesslog "WARN couldn't get sourcekey for source "$sourcename" from DB for "$runnumber"_"$filenum
242 fi
243 fi
244 fi
245
246 # build query to update runinfo in DB
247 query="UPDATE RunInfo SET "
248
249 # fill source key only if available
250 if ! [ "$sourcekey" = "" ]
251 then
252 query=$query" fSourceKey="$sourcekey", "
253 else
254 query=$query" fSourceKey=NULL, "
255 fi
256
257 # get information from tracking
258 if [ -e $trackingfile ]
259 then
260 # get statistics
261 trackingstats=`$factpath/fitsdump $trackingfile -s -c Time -c Ra -c Dec -c Zd -c Az --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null`
262 # RA
263 evaluatestatistics "Ra" $trackingstats
264 #echo $min"_"$max"_"$
265 if [ "$evaluation" != "" ]
266 then
267 if [ "$min" == "$max" ]
268 then
269 query=$query" fRightAscension="$mean
270 else
271 query=$query" fRightAscension=NULL"
272 printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")."
273 fi
274 # Declination
275 evaluatestatistics "Dec" $trackingstats
276 if [ "$decmin" == "$decmax" ]
277 then
278 query=$query", fDeclination="$mean
279 else
280 query=$query", fDeclination=NULL"
281 printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")."
282 fi
283 else
284 query=$query" fRightAscension=NULL"
285 query=$query", fDeclination=NULL"
286 fi
287 # Zd
288 evaluatestatistics "Zd" $trackingstats
289 if [ "$evaluation" != "" ]
290 then
291 query=$query", fZenithDistanceMin="$min
292 query=$query", fZenithDistanceMean="$mean
293 query=$query", fZenithDistanceMax="$max
294 else
295 query=$query", fZenithDistanceMin=NULL"
296 query=$query", fZenithDistanceMean=NULL"
297 query=$query", fZenithDistanceMax=NULL"
298 fi
299 # Az
300 evaluatestatistics "Az" $trackingstats
301 if [ "$evaluation" != "" ]
302 then
303 query=$query", fAzimuthMin="$min
304 query=$query", fAzimuthMean="$mean
305 query=$query", fAzimuthMax="$max
306 else
307 query=$query", fAzimuthMin=NULL"
308 query=$query", fAzimuthMean=NULL"
309 query=$query", fAzimuthMax=NULL"
310 fi
311 else
312 query=$query" fRightAscension=NULL"
313 query=$query", fDeclination=NULL"
314 query=$query", fZenithDistanceMin=NULL"
315 query=$query", fZenithDistanceMean=NULL"
316 query=$query", fZenithDistanceMax=NULL"
317 query=$query", fAzimuthMin=NULL"
318 query=$query", fAzimuthMean=NULL"
319 query=$query", fAzimuthMax=NULL"
320 fi
321
322 # get information from trigger
323 if [ -e $triggerratefile ]
324 then
325 # get statistics
326 triggerstats=`$factpath/fitsdump $triggerratefile -s -c Time -c TriggerRate --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null`
327 evaluatestatistics "TriggerRate" $triggerstats
328 if [ "$evaluation" != "" ]
329 then
330 query=$query", fTriggerRateMedian="$med
331 else
332 query=$query", fTriggerRateMedian=NULL"
333 fi
334 else
335 query=$query", fTriggerRateMedian=NULL"
336 fi
337
338 # get information from thresholds
339 if [ -e $thresholdfile ]
340 then
341 # get statistics
342 thresholdstats=`$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null`
343 evaluatestatistics "PatchThresh" $thresholdstats
344 if [ "$evaluation" = "" ]
345 then
346 thresholdstats=`$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null`
347 #echo "$factpath/fitsdump $thresholdfile -s -c Time -c PatchThresh --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null"
348 evaluatestatistics "PatchThresh" $thresholdstats
349 fi
350 if [ "$evaluation" != "" ]
351 then
352 query=$query", fThresholdMedian="$med
353 else
354 query=$query", fThresholdMedian=NULL"
355 fi
356 else
357 query=$query", fThresholdMedian=NULL"
358 fi
359
360 # get information from bias: U
361 if [ -e $biasvoltagefile ]
362 then
363 if [ $runnumber -gt 20120324 ]
364 then
365 biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c Uout --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null`
366 evaluatestatistics "Uout" $biasstats
367 if [ "$evaluation" = "" ]
368 then
369 biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c Uout --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null`
370 evaluatestatistics "Uout" $biasstats
371 fi
372 else
373 biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c U --filter='[1]<'${tstop}' && [1]>'${tstart} 2>/dev/null`
374 evaluatestatistics "U" $biasstats
375 if [ "$evaluation" = "" ]
376 then
377 biasstats=`$factpath/fitsdump $biasvoltagefile -s -c Time -c U --filter='[1]<'${tstop}' && [1]>'${tstart2} 2>/dev/null`
378 evaluatestatistics "U" $biasstats
379 fi
380 fi
381 if [ "$evaluation" != "" ]
382 then
383 query=$query", fBiasVoltageMedian="$med
384 else
385 query=$query", fBiasVoltageMedian=NULL"
386 fi
387 else
388 query=$query", fBiasVoltageMedian=NULL"
389 fi
390
391 # add where condition
392 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
393 #echo $query
394
395 # send query to DB
396 sendquery >/dev/null
397 done
398done
399
400finish
401
402
Note: See TracBrowser for help on using the repository browser.