source: trunk/DataCheck/Processing/FillAuxCurrents.sh@ 17587

Last change on this file since 17587 was 17581, checked in by Daniela Dorner, 11 years ago
implemented fz
  • Property svn:executable set to *
File size: 9.8 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillCurrents-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 getdates $certaindate
31else
32 # get all night
33 #getdates "all"
34 # get last 6 nights
35 getdates 6
36fi
37
38printprocesslog "INFO processing the following night(s): "${dates[@]}
39echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
40
41cd $mars
42
43# do filling of aux data
44for date in ${dates[@]}
45do
46 auxdir=$auxdata/$date
47 runnumber=`echo $date | sed -e 's/\///g'`
48
49 #if [ $runnumber -lt 20130301 ]
50 #then
51 # continue
52 #fi
53
54 # check if aux files are available from that night
55 if ! [ -d $auxdir ]
56 then
57 printprocesslog "INFO no data available in "$auxdir
58 continue
59 else
60 printprocesslog "INFO processing files in "$auxdir
61 fi
62
63 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
64 if ! [ -e $biasvoltagefile ]
65 then
66 printprocesslog "WARN "$biasvoltagefile" not found."
67 continue
68 else
69 biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
70 if [ $biasnumerrors -gt 0 ]
71 then
72 printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)."
73 fi
74 fi
75
76 biascurrentfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
77 if ! [ -e $biascurrentfile ]
78 then
79 printprocesslog "WARN "$biascurrentfile" not found."
80 continue
81 else
82 biascurrnumerrors=`fverify $biascurrentfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
83 if [ $biascurrnumerrors -gt 0 ]
84 then
85 printprocesslog "WARN for $biascurrentfile fverify returned "$biascurrnumerrors" error(s)."
86 fi
87 fi
88
89 # this file is needed for the calibration of the currents
90 feedbackcalfile=$auxdir/$runnumber.FEEDBACK_CALIBRATION.fits
91 if ! [ -e $feedbackcalfile ]
92 then
93 printprocesslog "WARN "$feedbackcalfile" not found."
94 continue
95 else
96 feedbacknumerrors=`fverify $feedbackcalfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
97 if [ $feedbacknumerrors -gt 0 ]
98 then
99 printprocesslog "WARN for $feedbackcalfile fverify returned "$feedbacknumerrors" error(s)."
100 fi
101 fi
102
103 calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
104 if ! [ -e $calcurrentsfile ]
105 then
106 calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
107 if ! [ -e $calcurrentsfile ]
108 then
109 printprocesslog "INFO run calibrate.C for night "$runnumber >> $logfile 2>&1
110 root -q -b -l fact/processing/calibrate.C\($runnumber\)
111 fi
112 fi
113 printprocesslog "INFO using calibrated currents from file "$calcurrentsfile
114
115 #calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
116 #calcurrentsfile=/scratch_nfs/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
117 #calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
118 if ! [ -e $calcurrentsfile ]
119 then
120 printprocesslog "WARN "$calcurrentsfile" not found."
121 continue
122 else
123 calnumerrors=`fverify $calcurrentsfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
124 if [ $calnumerrors -gt 0 ]
125 then
126 printprocesslog "WARN for $calcurrentsfile fverify returned "$calnumerrors" error(s)."
127 fi
128 fi
129
130
131 # get file numbers from DB
132 # but only for not-corrupted files
133 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
134 if [ "$doupdate" = "no" ]
135 then
136 query=$query" AND ISNULL(fCurrentsMedMean) "
137 fi
138
139 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
140 filenumbers=( `sendquery $query` )
141 if [ ${#filenumbers} -eq 0 ]
142 then
143 printprocesslog "INFO No files found in the DB for night "$date
144 continue
145 fi
146
147 # fill auxiliary information for files
148 for filenum in ${filenumbers[@]}
149 do
150 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
151 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
152 # get information from rawfile
153 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
154 if ! [ -e $rawfile ]
155 then
156 printprocesslog "ERROR: "$rawfile" not found."
157 continue
158 fi
159 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
160 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
161 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
162 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
163 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
164 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
165 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
166 then
167 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
168 continue
169 fi
170 # assuming that at least TSTARTI and TSTOPI are consistent
171 #echo $rawfile
172 #echo $tstarti
173 #echo $tstopi
174 #echo $tstartf
175 #echo $tstopf
176 if [ $tstarti -gt 30000 ]
177 then
178 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
179 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
180 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
181 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
182 else
183 tstart=`echo " $tstarti + $tstartf " | bc -l`
184 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
185 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
186 tstop=`echo " $tstopi + $tstopf " | bc -l`
187 fi
188
189 # build query to update runinfo in DB
190 query="UPDATE RunInfo SET "
191
192 # get information from fsc: T[31]
193 if [ -e $calcurrentsfile ] && [ $calnumerrors -eq 0 ]
194 then
195 #root -q -b -l fact/curavg.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) # | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'
196 #currents=( `root -q -b -l fact/curavg.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` )
197 currents=( `root -q -b -l fact/processing/currents.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
198 if [ "${currents[0]}" == "" ]
199 then
200 query=$query"fCurrentsMedMean=NULL"
201 else
202 query=$query"fCurrentsMedMean="${currents[0]}
203 fi
204 if [ "${currents[1]}" == "" ]
205 then
206 query=$query", fCurrentsMedRms=NULL"
207 else
208 query=$query", fCurrentsMedRms="${currents[1]}
209 fi
210 if [ "${currents[2]}" == "" ]
211 then
212 query=$query", fCurrentsDevMean=NULL"
213 else
214 query=$query", fCurrentsDevMean="${currents[2]}
215 fi
216 if [ "${currents[3]}" == "" ]
217 then
218 query=$query", fCurrentsDevRms=NULL"
219 else
220 query=$query", fCurrentsDevRms="${currents[3]}
221 fi
222 if [ "${currents[4]}" == "" ]
223 then
224 query=$query", fCurrentsMedMeanBeg=NULL"
225 else
226 query=$query", fCurrentsMedMeanBeg="${currents[4]}
227 fi
228 if [ "${currents[5]}" == "" ]
229 then
230 query=$query", fCurrentsMedMeanEnd=NULL"
231 else
232 query=$query", fCurrentsMedMeanEnd="${currents[5]}
233 fi
234 if [ "${currents[6]}" == "" ]
235 then
236 query=$query", fCurrentsDiffToPrediction=NULL"
237 else
238 query=$query", fCurrentsDiffToPrediction="${currents[6]}
239 fi
240 if [ "${currents[7]}" == "" ]
241 then
242 query=$query", fCurrentsRelDiffToPrediction=NULL"
243 else
244 query=$query", fCurrentsRelDiffToPrediction="${currents[7]}
245 fi
246 if [ "${currents[8]}" == "" ]
247 then
248 query=$query", fCurrentsLineRms=NULL"
249 else
250 query=$query", fCurrentsLineRms="${currents[8]}
251 fi
252 if [ "${currents[9]}" == "" ]
253 then
254 query=$query", fCurrentsRelLineRms=NULL"
255 else
256 query=$query", fCurrentsRelLineRms="${currents[9]}
257 fi
258 else
259 query=$query" fCurrentsMedMean=NULL"
260 query=$query", fCurrentsMedRms=NULL"
261 query=$query", fCurrentsDevMean=NULL"
262 query=$query", fCurrentsDevRms=NULL"
263 query=$query", fCurrentsMedMeanBeg=NULL"
264 query=$query", fCurrentsMedMeanEnd=NULL"
265 query=$query", fCurrentsDiffToPrediction=NULL"
266 query=$query", fCurrentsRelDiffToPrediction=NULL"
267 query=$query", fCurrentsLineRms=NULL"
268 query=$query", fCurrentsRelLineRms=NULL"
269 fi
270
271 # add where condition
272 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
273
274 #echo $query
275 # send query to DB
276 sendquery >/dev/null
277 done
278done
279
280finish
281
282
Note: See TracBrowser for help on using the repository browser.