source: trunk/DataCheck/Processing/FillAuxCurrents.sh@ 17440

Last change on this file since 17440 was 17092, checked in by Daniela Dorner, 11 years ago
new macro paths, usage of getdates function, prepared script to be used in RunMoreNights.sh, added 4 new variables, run calibrate.C only if really needed
  • Property svn:executable set to *
File size: 9.7 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillCurrents-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 getdates $certaindate
31else
32 # get all night
33 #getdates "all"
34 # get last 6 nights
35 getdates 6
36fi
37
38printprocesslog "INFO processing the following night(s): "${dates[@]}
39echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
40
41cd $mars
42
43# do filling of aux data
44for date in ${dates[@]}
45do
46 auxdir=$auxdata/$date
47 runnumber=`echo $date | sed -e 's/\///g'`
48
49 #if [ $runnumber -lt 20130301 ]
50 #then
51 # continue
52 #fi
53
54 # check if aux files are available from that night
55 if ! [ -d $auxdir ]
56 then
57 printprocesslog "INFO no data available in "$auxdir
58 continue
59 else
60 printprocesslog "INFO processing files in "$auxdir
61 fi
62
63 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
64 if ! [ -e $biasvoltagefile ]
65 then
66 printprocesslog "WARN "$biasvoltagefile" not found."
67 continue
68 else
69 biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
70 if [ $biasnumerrors -gt 0 ]
71 then
72 printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)."
73 fi
74 fi
75
76 biascurrentfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
77 if ! [ -e $biascurrentfile ]
78 then
79 printprocesslog "WARN "$biascurrentfile" not found."
80 continue
81 else
82 biascurrnumerrors=`fverify $biascurrentfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
83 if [ $biascurrnumerrors -gt 0 ]
84 then
85 printprocesslog "WARN for $biascurrentfile fverify returned "$biascurrnumerrors" error(s)."
86 fi
87 fi
88
89 feedbackcalfile=$auxdir/$runnumber.FEEDBACK_CALIBRATION.fits
90 if ! [ -e $feedbackcalfile ]
91 then
92 printprocesslog "WARN "$feedbackcalfile" not found."
93 continue
94 else
95 feedbacknumerrors=`fverify $feedbackcalfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
96 if [ $feedbacknumerrors -gt 0 ]
97 then
98 printprocesslog "WARN for $feedbackcalfile fverify returned "$feedbacknumerrors" error(s)."
99 fi
100 fi
101
102 calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
103 if ! [ -e $calcurrentsfile ]
104 then
105 calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
106 if ! [ -e $calcurrentsfile ]
107 then
108 printprocesslog "INFO run calibrate.C for night "$runnumber >> $logfile 2>&1
109 root -q -b -l fact/processing/calibrate.C\($runnumber\)
110 fi
111 fi
112 printprocesslog "INFO using calibrated currents from file "$calcurrentsfile
113
114 #calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
115 #calcurrentsfile=/scratch_nfs/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
116 #calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
117 if ! [ -e $calcurrentsfile ]
118 then
119 printprocesslog "WARN "$calcurrentsfile" not found."
120 continue
121 else
122 calnumerrors=`fverify $calcurrentsfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
123 if [ $calnumerrors -gt 0 ]
124 then
125 printprocesslog "WARN for $calcurrentsfile fverify returned "$calnumerrors" error(s)."
126 fi
127 fi
128
129
130 # get file numbers from DB
131 # but only for not-corrupted files
132 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
133 if [ "$doupdate" = "no" ]
134 then
135 query=$query" AND ISNULL(fCurrentsMedMean) "
136 fi
137
138 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
139 filenumbers=( `sendquery $query` )
140 if [ ${#filenumbers} -eq 0 ]
141 then
142 printprocesslog "INFO No files found in the DB for night "$date
143 continue
144 fi
145
146 # fill auxiliary information for files
147 for filenum in ${filenumbers[@]}
148 do
149 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
150 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
151 # get information from rawfile
152 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.gz
153 if ! [ -e $rawfile ]
154 then
155 printprocesslog "ERROR: "$rawfile" not found."
156 continue
157 fi
158 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
159 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
160 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
161 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
162 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
163 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
164 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
165 then
166 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
167 continue
168 fi
169 # assuming that at least TSTARTI and TSTOPI are consistent
170 #echo $rawfile
171 #echo $tstarti
172 #echo $tstopi
173 #echo $tstartf
174 #echo $tstopf
175 if [ $tstarti -gt 30000 ]
176 then
177 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
178 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
179 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
180 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
181 else
182 tstart=`echo " $tstarti + $tstartf " | bc -l`
183 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
184 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
185 tstop=`echo " $tstopi + $tstopf " | bc -l`
186 fi
187
188 # build query to update runinfo in DB
189 query="UPDATE RunInfo SET "
190
191 # get information from fsc: T[31]
192 if [ -e $calcurrentsfile ] && [ $calnumerrors -eq 0 ]
193 then
194 #root -q -b -l fact/curavg.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) # | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'
195 #currents=( `root -q -b -l fact/curavg.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` )
196 currents=( `root -q -b -l fact/processing/currents.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
197 if [ "${currents[0]}" == "" ]
198 then
199 query=$query"fCurrentsMedMean=NULL"
200 else
201 query=$query"fCurrentsMedMean="${currents[0]}
202 fi
203 if [ "${currents[1]}" == "" ]
204 then
205 query=$query", fCurrentsMedRms=NULL"
206 else
207 query=$query", fCurrentsMedRms="${currents[1]}
208 fi
209 if [ "${currents[2]}" == "" ]
210 then
211 query=$query", fCurrentsDevMean=NULL"
212 else
213 query=$query", fCurrentsDevMean="${currents[2]}
214 fi
215 if [ "${currents[3]}" == "" ]
216 then
217 query=$query", fCurrentsDevRms=NULL"
218 else
219 query=$query", fCurrentsDevRms="${currents[3]}
220 fi
221 if [ "${currents[4]}" == "" ]
222 then
223 query=$query", fCurrentsMedMeanBeg=NULL"
224 else
225 query=$query", fCurrentsMedMeanBeg="${currents[4]}
226 fi
227 if [ "${currents[5]}" == "" ]
228 then
229 query=$query", fCurrentsMedMeanEnd=NULL"
230 else
231 query=$query", fCurrentsMedMeanEnd="${currents[5]}
232 fi
233 if [ "${currents[6]}" == "" ]
234 then
235 query=$query", fCurrentsDiffToPrediction=NULL"
236 else
237 query=$query", fCurrentsDiffToPrediction="${currents[6]}
238 fi
239 if [ "${currents[7]}" == "" ]
240 then
241 query=$query", fCurrentsRelDiffToPrediction=NULL"
242 else
243 query=$query", fCurrentsRelDiffToPrediction="${currents[7]}
244 fi
245 if [ "${currents[8]}" == "" ]
246 then
247 query=$query", fCurrentsLineRms=NULL"
248 else
249 query=$query", fCurrentsLineRms="${currents[8]}
250 fi
251 if [ "${currents[9]}" == "" ]
252 then
253 query=$query", fCurrentsRelLineRms=NULL"
254 else
255 query=$query", fCurrentsRelLineRms="${currents[9]}
256 fi
257 else
258 query=$query" fCurrentsMedMean=NULL"
259 query=$query", fCurrentsMedRms=NULL"
260 query=$query", fCurrentsDevMean=NULL"
261 query=$query", fCurrentsDevRms=NULL"
262 query=$query", fCurrentsMedMeanBeg=NULL"
263 query=$query", fCurrentsMedMeanEnd=NULL"
264 query=$query", fCurrentsDiffToPrediction=NULL"
265 query=$query", fCurrentsRelDiffToPrediction=NULL"
266 query=$query", fCurrentsLineRms=NULL"
267 query=$query", fCurrentsRelLineRms=NULL"
268 fi
269
270 # add where condition
271 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
272
273 #echo $query
274 # send query to DB
275 sendquery >/dev/null
276 done
277done
278
279finish
280
281
Note: See TracBrowser for help on using the repository browser.