source: trunk/DataCheck/FillAuxData.sh@ 12923

Last change on this file since 12923 was 12789, checked in by Daniela Dorner, 13 years ago
included further checks and fill more information; common logfile
  • Property svn:executable set to *
File size: 16.3 KB
Line 
1#!/bin/bash
2
3# missing:
4# vferify of files
5# only treat file if it is there
6
7# mjd of 1970-01-01
8# needed in this script as for 1 day the mjd in the aux files are inconsistent
9mjdref=40587
10
11# option
12doupdate="yes" # update all entries (needed when new fields have been added)
13doupdate="no" # fill only entries which are not yet existing (default)
14
15source `dirname $0`/Sourcefile.sh
16printprocesslog "INFO starting $0 with option doupdate="$doupdate
17
18# setup to use ftools
19source $HEADAS/headas-init.sh
20
21pwfile=`dirname $0`/.pw
22password=`cat $pwfile 2>/dev/null`
23if [ "$password" == "" ]
24then
25 echo "please insert password in $pwfile"
26 printprocesslog "ERROR password for DB access in $pwfile missing"
27 finish
28fi
29
30# check if software is available
31if ! ls $factpath/fitsdump >/dev/null 2>&1
32then
33 printprocesslog "ERROR "$factpath"/fitsdump is not available."
34 finish
35fi
36
37# get last 2 nights
38dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
39#dates=( `date +%Y/%m/%d --date="-12hour"` )
40dates=( "2011/12/07" )
41# problems with 2011/11/29
42dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw
43printprocesslog "INFO processing the following night: "${dates[@]}
44
45# do filling of aux data
46for date in ${dates[@]}
47do
48 auxdir=$auxdata/$date
49 runnumber=`echo $date | sed -e 's/\///g'`
50 # check if data are available from that night
51 if ! [ -d $auxdir ]
52 then
53 printprocesslog "INFO no data available in "$auxdir
54 continue
55 else
56 printprocesslog "INFO processing files in "$auxdir
57 fi
58
59 # get daily fits files
60 trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
61 if ! [ -e $trackingfile ]
62 then
63 printprocesslog "WARN "$trackingfile" not found."
64 fi
65 triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
66 if ! [ -e $triggerratefile ]
67 then
68 printprocesslog "WARN "$triggerratefile" not found."
69 fi
70 thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
71 if ! [ -e $thresholdfile ]
72 then
73 printprocesslog "WARN "$thresholdfile" not found."
74 fi
75 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
76 if ! [ -e $biasvoltagefile ]
77 then
78 printprocesslog "WARN "$biasvoltagefile" not found."
79 fi
80 biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
81 if ! [ -e $biascurrentsfile ]
82 then
83 printprocesslog "WARN "$biascurrentsfile" not found."
84 fi
85
86
87
88 # get file numbers from DB
89 query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber
90 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query1
91 filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>/dev/null` )
92 for filenum in ${filenumbers[@]}
93 do
94 printprocesslog "INFO processing file number "$filenum
95
96 # get information from rawfile
97 rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits
98 if ! [ -e $rawfile ]
99 then
100 printprocesslog "ERROR: "$rawfile" not found."
101 continue
102 fi
103 checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
104 if [ "$checkfitsfile" == "" ]
105 then
106 numfitserrors=1
107 printprocesslog "WARN: "$rawfile" probably corrupted."
108 continue
109 fi
110 runtype=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
111 mjdrefraw=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
112 tstarti=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
113 tstartf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
114 tstopi=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
115 tstopf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
116 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ]
117 then
118 printprocesslog "WARN: "$rawfile" has start or stop time = 0."
119 continue
120 fi
121 tstart=`echo " $tstarti + $tstartf + $mjdrefraw " | bc -l`
122 tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.00011574 " | bc -l` # 10 sec
123 #tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.000023148 " | bc -l` # 2 sec
124 tstop=`echo " $tstopi + $tstopf + $mjdrefraw " | bc -l`
125 # calculate start and stop times
126 # 2011/11/22 MJDREF in DRIVE empty, Time > 55000
127 # 2011/11/23 MJDREF in DRIVE not empty, Time > 55000
128 # 2011/11/24 MJDREF in DRIVE not empty, Time > 15000
129 # raw files
130 # 2011/11/21 no MJDREF
131 # 2011/11/22 MJDREF
132 if [ $runnumber -eq 20111123 ]
133 then
134 # add mjdref for days were aux files were inconsistent
135 tstart=`echo " $tstart + $mjdref " | bc -l`
136 tstart2=`echo " $tstart2 + $mjdref " | bc -l`
137 tstop=`echo " $tstop + $mjdref " | bc -l`
138 fi
139
140 # build query to update runinfo in DB
141 query0="UPDATE RunInfo SET "
142 queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
143
144 # get information from tracking
145 if [ -e $trackingfile ]
146 then
147 ramin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
148 ramax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
149 ramean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
150 decmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
151 decmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
152 decmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
153 zdmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
154 zdmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
155 zdmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
156 azmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
157 azmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
158 azmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
159 # RA
160 if [ "$ramin" == "$ramax" ] && [ "$ramean" != "0" ]
161 then
162 query0=$query0" fRightAscension="$ramean
163 else
164 query0=$query0" fRightAscension=NULL"
165 fi
166 # Declination
167 if [ "$decmin" == "$decmax" ] && [ "$decmean" != "0" ]
168 then
169 query0=$query0", fDeclination="$decmean
170 else
171 query0=$query0", fDeclination=NULL"
172 fi
173 # Zd
174 if [ "$zdmin" != "" ] && [ "$zdmean" != "0" ]
175 then
176 query0=$query0", fZenithDistanceMin="$zdmin
177 else
178 query0=$query0", fZenithDistanceMin=NULL"
179 fi
180 if [ "$zdmean" != "" ] && [ "$zdmean" != "0" ]
181 then
182 query0=$query0", fZenithDistanceMean="$zdmean
183 else
184 query0=$query0", fZenithDistanceMean=NULL"
185 fi
186 if [ "$zdmax" != "" ] && [ "$zdmean" != "0" ]
187 then
188 query0=$query0", fZenithDistanceMax="$zdmax
189 else
190 query0=$query0", fZenithDistanceMax=NULL"
191 fi
192 # Az
193 if [ "$azmin" != "" ] && [ "$azmean" != "0" ]
194 then
195 query0=$query0", fAzimuthMin="$azmin
196 else
197 query0=$query0", fAzimuthMin=NULL"
198 fi
199 if [ "$azmean" != "" ] && [ "$azmean" != "0" ]
200 then
201 query0=$query0", fAzimuthMean="$azmean
202 else
203 query0=$query0", fAzimuthMean=NULL"
204 fi
205 if [ "$azmax" != "" ] && [ "$azmean" != "0" ]
206 then
207 query0=$query0", fAzimuthMax="$azmax
208 else
209 query0=$query0", fAzimuthMax=NULL"
210 fi
211 fi
212
213 # get information from trigger
214 if ! [ -e $triggerratefile ]
215 then
216 ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
217 ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
218 ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
219 ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
220 if [ "$ratemedian" != "0" ]
221 then
222 query0=$query0", fTriggerRateMedian="$ratemedian
223 else
224 query0=$query0", fTriggerRateMedian=NULL"
225 fi
226 fi
227
228 # get information from thresholds
229 if ! [ -e $thresholdfile ]
230 then
231 threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
232 threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
233 threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
234 threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
235 if [ "$threshmean" == "0" ]
236 then
237 threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
238 threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
239 threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
240 threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
241 # is this taking into account the 160 values in a row?
242 fi
243 query0=$query0", fThresholdMedian="$threshmedian
244 fi
245
246# # get information from bias: Uref
247# biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
248# biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
249# biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
250# biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
251# if [ "$biasvoltmean" == "0" ]
252# then
253# biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
254# biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
255# biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
256# biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
257# # is this taking into account the 160 values in a row?
258# fi
259
260 # get information from bias: U
261 if ! [ -e $biasvoltagefile ]
262 then
263 biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
264 biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
265 biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
266 biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
267 if [ "$biasvoltmean" == "0" ]
268 then
269 biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
270 biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
271 biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
272 biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
273 # is this taking into account the 160 values in a row?
274 fi
275 query0=$query0", fBiasVoltageMedian="$biasvoltmedian
276 fi
277
278 query0=$query0" "$queryend
279 printprocesslog "DEBUG update DB entry with QUERY: "$query0
280 # send query to DB
281 if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >/dev/null 2>&1
282 then
283 printprocesslog "ERROR insert/update of "$runnumber" "$filenum" to mysql failed."
284 fi
285 done
286done
287
288finish
289
290
Note: See TracBrowser for help on using the repository browser.