source: trunk/DataCheck/Processing/FillAuxCurrents.sh@ 20109

Last change on this file since 20109 was 19524, checked in by Daniela Dorner, 6 years ago
removed not needed line
  • Property svn:executable set to *
File size: 9.8 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillCurrents-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 getdates $certaindate
31else
32 # get all night
33 #getdates "all"
34 # get last 6 nights
35 getdates 6
36fi
37
38
39
40printprocesslog "INFO processing the following night(s): "${dates[@]}
41echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
42
43cd $mars
44
45# do filling of aux data
46for date in ${dates[@]}
47do
48 auxdir=$auxdata/$date
49 runnumber=`echo $date | sed -e 's/\///g'`
50
51 #if [ $runnumber -lt 20130301 ]
52 #then
53 # continue
54 #fi
55
56 # get file numbers from DB
57 # but only for not-corrupted files
58 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
59 if [ "$doupdate" = "no" ]
60 then
61 query=$query" AND ISNULL(fCurrentsMedMean) "
62 fi
63 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
64 filenumbers=( `sendquery $query` )
65 # proceed only if there are files available
66 if [ ${#filenumbers} -eq 0 ]
67 then
68 printprocesslog "INFO No files found in the DB for night "$date
69 continue
70 fi
71
72 # check if aux files are available from that night
73 if ! [ -d $auxdir ]
74 then
75 printprocesslog "INFO no data available in "$auxdir
76 continue
77 else
78 printprocesslog "INFO processing files in "$auxdir
79 fi
80
81 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
82 #if ! [ -e $biasvoltagefile ]
83 if ! check_file_avail $biasvoltagefile
84 then
85 #printprocesslog "WARN "$biasvoltagefile" not found."
86 continue
87 else
88 biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
89 if [ $biasnumerrors -gt 0 ]
90 then
91 printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)."
92 fi
93 fi
94
95 biascurrentfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
96 #if ! [ -e $biascurrentfile ]
97 if ! check_file_avail $biascurrentfile
98 then
99 #printprocesslog "WARN "$biascurrentfile" not found."
100 continue
101 else
102 biascurrnumerrors=`fverify $biascurrentfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
103 if [ $biascurrnumerrors -gt 0 ]
104 then
105 printprocesslog "WARN for $biascurrentfile fverify returned "$biascurrnumerrors" error(s)."
106 fi
107 fi
108
109 # this file is needed for the calibration of the currents
110 feedbackcalfile=$auxdir/$runnumber.FEEDBACK_CALIBRATION.fits
111 #if ! [ -e $feedbackcalfile ]
112 if ! check_file_avail $feedbackcalfile
113 then
114 #printprocesslog "WARN "$feedbackcalfile" not found."
115 continue
116 else
117 feedbacknumerrors=`fverify $feedbackcalfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
118 if [ $feedbacknumerrors -gt 0 ]
119 then
120 printprocesslog "WARN for $feedbackcalfile fverify returned "$feedbacknumerrors" error(s)."
121 fi
122 fi
123
124 calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
125 #if ! [ -e $calcurrentsfile ]
126 if ! check_file_avail $calcurrentsfile
127 then
128 calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
129 if ! [ -e $calcurrentsfile ]
130 then
131 printprocesslog "INFO run calibrate.C for night "$runnumber >> $logfile 2>&1
132 root -q -b -l fact/processing/calibrate.C\($runnumber\)
133 fi
134 fi
135 printprocesslog "INFO using calibrated currents from file "$calcurrentsfile
136
137 #calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
138 #calcurrentsfile=/scratch_nfs/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
139 #calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
140 #if ! [ -e $calcurrentsfile ]
141 if ! check_file_avail $calcurrentsfile
142 then
143 #printprocesslog "WARN "$calcurrentsfile" not found."
144 continue
145 else
146 calnumerrors=`fverify $calcurrentsfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
147 if [ $calnumerrors -gt 0 ]
148 then
149 printprocesslog "WARN for $calcurrentsfile fverify returned "$calnumerrors" error(s)."
150 fi
151 fi
152
153 # fill auxiliary information for files
154 for filenum in ${filenumbers[@]}
155 do
156 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
157 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
158 # get information from rawfile
159 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
160 #if ! [ -e $rawfile ]
161 if ! check_file_avail $rawfile
162 then
163# if [ $runnumber -lt $checknight ]
164# then
165# printprocesslog "WARN "$rawfile" not found."
166# else
167# printprocesslog "INFO "$rawfile" not found."
168# fi
169 continue
170 fi
171 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
172 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
173 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
174 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
175 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
176 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
177 then
178 printprocesslog "WARN "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
179 continue
180 fi
181 # assuming that at least TSTARTI and TSTOPI are consistent
182 if [ $tstarti -gt 30000 ]
183 then
184 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
185 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
186 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
187 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
188 else
189 tstart=`echo " $tstarti + $tstartf " | bc -l`
190 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
191 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
192 tstop=`echo " $tstopi + $tstopf " | bc -l`
193 fi
194
195 # build query to update runinfo in DB
196 query="UPDATE RunInfo SET "
197
198 # get information from fsc: T[31]
199 if [ -e $calcurrentsfile ] && [ $calnumerrors -eq 0 ]
200 then
201 currents=( `root -q -b -l fact/processing/currents.C\("\"$calcurrentsfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
202 if [ "${currents[0]}" == "" ]
203 then
204 query=$query"fCurrentsMedMean=NULL"
205 else
206 query=$query"fCurrentsMedMean="${currents[0]}
207 fi
208 if [ "${currents[1]}" == "" ]
209 then
210 query=$query", fCurrentsMedRms=NULL"
211 else
212 query=$query", fCurrentsMedRms="${currents[1]}
213 fi
214 if [ "${currents[2]}" == "" ]
215 then
216 query=$query", fCurrentsDevMean=NULL"
217 else
218 query=$query", fCurrentsDevMean="${currents[2]}
219 fi
220 if [ "${currents[3]}" == "" ]
221 then
222 query=$query", fCurrentsDevRms=NULL"
223 else
224 query=$query", fCurrentsDevRms="${currents[3]}
225 fi
226 if [ "${currents[4]}" == "" ]
227 then
228 query=$query", fCurrentsMedMeanBeg=NULL"
229 else
230 query=$query", fCurrentsMedMeanBeg="${currents[4]}
231 fi
232 if [ "${currents[5]}" == "" ]
233 then
234 query=$query", fCurrentsMedMeanEnd=NULL"
235 else
236 query=$query", fCurrentsMedMeanEnd="${currents[5]}
237 fi
238 if [ "${currents[6]}" == "" ]
239 then
240 query=$query", fCurrentsDiffToPrediction=NULL"
241 else
242 query=$query", fCurrentsDiffToPrediction="${currents[6]}
243 fi
244 if [ "${currents[7]}" == "" ]
245 then
246 query=$query", fCurrentsRelDiffToPrediction=NULL"
247 else
248 query=$query", fCurrentsRelDiffToPrediction="${currents[7]}
249 fi
250 if [ "${currents[8]}" == "" ]
251 then
252 query=$query", fCurrentsLineRms=NULL"
253 else
254 query=$query", fCurrentsLineRms="${currents[8]}
255 fi
256 if [ "${currents[9]}" == "" ]
257 then
258 query=$query", fCurrentsRelLineRms=NULL"
259 else
260 query=$query", fCurrentsRelLineRms="${currents[9]}
261 fi
262 else
263 query=$query" fCurrentsMedMean=NULL"
264 query=$query", fCurrentsMedRms=NULL"
265 query=$query", fCurrentsDevMean=NULL"
266 query=$query", fCurrentsDevRms=NULL"
267 query=$query", fCurrentsMedMeanBeg=NULL"
268 query=$query", fCurrentsMedMeanEnd=NULL"
269 query=$query", fCurrentsDiffToPrediction=NULL"
270 query=$query", fCurrentsRelDiffToPrediction=NULL"
271 query=$query", fCurrentsLineRms=NULL"
272 query=$query", fCurrentsRelLineRms=NULL"
273 fi
274
275 # add where condition
276 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
277
278 # send query to DB
279 sendquery >/dev/null
280 done
281done
282
283finish
284
285
Note: See TracBrowser for help on using the repository browser.