source: trunk/DataCheck/Processing/FillAuxCurrents.sh@ 18259

Last change on this file since 18259 was 17907, checked in by Daniela Dorner, 10 years ago
included and/or improved check of variable certaindate
  • Property svn:executable set to *
File size: 10.1 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillCurrents-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
31 if [ "$checkstring" = "" ]
32 then
33 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
34 finish
35 fi
36 getdates $certaindate
37else
38 # get all night
39 #getdates "all"
40 # get last 6 nights
41 getdates 6
42fi
43
44printprocesslog "INFO processing the following night(s): "${dates[@]}
45echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
46
47cd $mars
48
49# do filling of aux data
50for date in ${dates[@]}
51do
52 auxdir=$auxdata/$date
53 runnumber=`echo $date | sed -e 's/\///g'`
54
55 #if [ $runnumber -lt 20130301 ]
56 #then
57 # continue
58 #fi
59
60 # get file numbers from DB
61 # but only for not-corrupted files
62 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
63 if [ "$doupdate" = "no" ]
64 then
65 query=$query" AND ISNULL(fCurrentsMedMean) "
66 fi
67 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
68 filenumbers=( `sendquery $query` )
69 # proceed only if there are files available
70 if [ ${#filenumbers} -eq 0 ]
71 then
72 printprocesslog "INFO No files found in the DB for night "$date
73 continue
74 fi
75
76 # check if aux files are available from that night
77 if ! [ -d $auxdir ]
78 then
79 printprocesslog "INFO no data available in "$auxdir
80 continue
81 else
82 printprocesslog "INFO processing files in "$auxdir
83 fi
84
85 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
86 if ! [ -e $biasvoltagefile ]
87 then
88 printprocesslog "WARN "$biasvoltagefile" not found."
89 continue
90 else
91 biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
92 if [ $biasnumerrors -gt 0 ]
93 then
94 printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)."
95 fi
96 fi
97
98 biascurrentfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
99 if ! [ -e $biascurrentfile ]
100 then
101 printprocesslog "WARN "$biascurrentfile" not found."
102 continue
103 else
104 biascurrnumerrors=`fverify $biascurrentfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
105 if [ $biascurrnumerrors -gt 0 ]
106 then
107 printprocesslog "WARN for $biascurrentfile fverify returned "$biascurrnumerrors" error(s)."
108 fi
109 fi
110
111 # this file is needed for the calibration of the currents
112 feedbackcalfile=$auxdir/$runnumber.FEEDBACK_CALIBRATION.fits
113 if ! [ -e $feedbackcalfile ]
114 then
115 printprocesslog "WARN "$feedbackcalfile" not found."
116 continue
117 else
118 feedbacknumerrors=`fverify $feedbackcalfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
119 if [ $feedbacknumerrors -gt 0 ]
120 then
121 printprocesslog "WARN for $feedbackcalfile fverify returned "$feedbacknumerrors" error(s)."
122 fi
123 fi
124
125 calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
126 if ! [ -e $calcurrentsfile ]
127 then
128 calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
129 if ! [ -e $calcurrentsfile ]
130 then
131 printprocesslog "INFO run calibrate.C for night "$runnumber >> $logfile 2>&1
132 root -q -b -l fact/processing/calibrate.C\($runnumber\)
133 fi
134 fi
135 printprocesslog "INFO using calibrated currents from file "$calcurrentsfile
136
137 #calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
138 #calcurrentsfile=/scratch_nfs/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
139 #calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
140 if ! [ -e $calcurrentsfile ]
141 then
142 printprocesslog "WARN "$calcurrentsfile" not found."
143 continue
144 else
145 calnumerrors=`fverify $calcurrentsfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
146 if [ $calnumerrors -gt 0 ]
147 then
148 printprocesslog "WARN for $calcurrentsfile fverify returned "$calnumerrors" error(s)."
149 fi
150 fi
151
152 # fill auxiliary information for files
153 for filenum in ${filenumbers[@]}
154 do
155 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
156 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
157 # get information from rawfile
158 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
159 if ! [ -e $rawfile ]
160 then
161 printprocesslog "ERROR: "$rawfile" not found."
162 continue
163 fi
164 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
165 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
166 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
167 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
168 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
169 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
170 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
171 then
172 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
173 continue
174 fi
175 # assuming that at least TSTARTI and TSTOPI are consistent
176 #echo $rawfile
177 #echo $tstarti
178 #echo $tstopi
179 #echo $tstartf
180 #echo $tstopf
181 if [ $tstarti -gt 30000 ]
182 then
183 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
184 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
185 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
186 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
187 else
188 tstart=`echo " $tstarti + $tstartf " | bc -l`
189 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
190 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
191 tstop=`echo " $tstopi + $tstopf " | bc -l`
192 fi
193
194 # build query to update runinfo in DB
195 query="UPDATE RunInfo SET "
196
197 # get information from fsc: T[31]
198 if [ -e $calcurrentsfile ] && [ $calnumerrors -eq 0 ]
199 then
200 #root -q -b -l fact/curavg.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) # | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'
201 #currents=( `root -q -b -l fact/curavg.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` )
202 currents=( `root -q -b -l fact/processing/currents.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
203 if [ "${currents[0]}" == "" ]
204 then
205 query=$query"fCurrentsMedMean=NULL"
206 else
207 query=$query"fCurrentsMedMean="${currents[0]}
208 fi
209 if [ "${currents[1]}" == "" ]
210 then
211 query=$query", fCurrentsMedRms=NULL"
212 else
213 query=$query", fCurrentsMedRms="${currents[1]}
214 fi
215 if [ "${currents[2]}" == "" ]
216 then
217 query=$query", fCurrentsDevMean=NULL"
218 else
219 query=$query", fCurrentsDevMean="${currents[2]}
220 fi
221 if [ "${currents[3]}" == "" ]
222 then
223 query=$query", fCurrentsDevRms=NULL"
224 else
225 query=$query", fCurrentsDevRms="${currents[3]}
226 fi
227 if [ "${currents[4]}" == "" ]
228 then
229 query=$query", fCurrentsMedMeanBeg=NULL"
230 else
231 query=$query", fCurrentsMedMeanBeg="${currents[4]}
232 fi
233 if [ "${currents[5]}" == "" ]
234 then
235 query=$query", fCurrentsMedMeanEnd=NULL"
236 else
237 query=$query", fCurrentsMedMeanEnd="${currents[5]}
238 fi
239 if [ "${currents[6]}" == "" ]
240 then
241 query=$query", fCurrentsDiffToPrediction=NULL"
242 else
243 query=$query", fCurrentsDiffToPrediction="${currents[6]}
244 fi
245 if [ "${currents[7]}" == "" ]
246 then
247 query=$query", fCurrentsRelDiffToPrediction=NULL"
248 else
249 query=$query", fCurrentsRelDiffToPrediction="${currents[7]}
250 fi
251 if [ "${currents[8]}" == "" ]
252 then
253 query=$query", fCurrentsLineRms=NULL"
254 else
255 query=$query", fCurrentsLineRms="${currents[8]}
256 fi
257 if [ "${currents[9]}" == "" ]
258 then
259 query=$query", fCurrentsRelLineRms=NULL"
260 else
261 query=$query", fCurrentsRelLineRms="${currents[9]}
262 fi
263 else
264 query=$query" fCurrentsMedMean=NULL"
265 query=$query", fCurrentsMedRms=NULL"
266 query=$query", fCurrentsDevMean=NULL"
267 query=$query", fCurrentsDevRms=NULL"
268 query=$query", fCurrentsMedMeanBeg=NULL"
269 query=$query", fCurrentsMedMeanEnd=NULL"
270 query=$query", fCurrentsDiffToPrediction=NULL"
271 query=$query", fCurrentsRelDiffToPrediction=NULL"
272 query=$query", fCurrentsLineRms=NULL"
273 query=$query", fCurrentsRelLineRms=NULL"
274 fi
275
276 # add where condition
277 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
278
279 #echo $query
280 # send query to DB
281 sendquery >/dev/null
282 done
283done
284
285finish
286
287
Note: See TracBrowser for help on using the repository browser.