source: trunk/DataCheck/Processing/FillAuxMagic.sh@ 19486

Last change on this file since 19486 was 19484, checked in by Daniela Dorner, 6 years ago
added check for cases where both the rawfile is not in the archive and the information not in the DB
  • Property svn:executable set to *
File size: 9.9 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillAuxTemp-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 getdates $certaindate
31 echo `date`": processing the following night(s): "${dates[@]}
32else
33 # get all night
34 #getdates "all"
35 # get last 6 nights
36 getdates 6
37fi
38
39
40
41printprocesslog "INFO processing the following night(s): "${dates[@]}
42echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
43
44cd $mars
45
46# do filling of aux data
47for date in ${dates[@]}
48do
49 auxdir=$auxdata/$date
50 runnumber=`echo $date | sed -e 's/\///g'`
51 if [ $runnumber -lt 20120328 ]
52 then
53 continue
54 fi
55
56 # check if aux files are available from that night
57 if ! [ -d $auxdir ]
58 then
59 printprocesslog "INFO no data available in "$auxdir
60 continue
61 else
62 printprocesslog "INFO processing files in "$auxdir
63 fi
64
65 # get file numbers from DB
66 # but only for not-corrupted files
67 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
68 if [ "$doupdate" = "no" ]
69 then
70 query=$query" AND ISNULL(fCameraTempMean) "
71 fi
72 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
73 filenumbers=( `sendquery $query` )
74 # proceed only if there are files available
75 if [ ${#filenumbers} -eq 0 ]
76 then
77 printprocesslog "INFO No files found in the DB for night "$date
78 continue
79 fi
80
81 magicweatherfile=$auxdir/$runnumber.MAGIC_WEATHER_DATA.fits
82 if ! [ -e $magicweatherfile ]
83 then
84 printprocesslog "WARN "$magicweatherfile" not found."
85 else
86 weathernumerrors=`fverify $magicweatherfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
87 if [ $weathernumerrors -gt 0 ]
88 then
89 printprocesslog "WARN for $magicweatherfile fverify returned "$weathernumerrors" error(s)."
90 fi
91 fi
92
93 # fill auxiliary information for files
94 for filenum in ${filenumbers[@]}
95 do
96 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
97 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
98 # get information from rawfile
99 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
100 if ! [ -e $rawfile ]
101 then
102 if [ $runnumber -lt $checknight ]
103 then
104 printprocesslog "WARN "$rawfile" not found."
105 else
106 printprocesslog "INFO "$rawfile" not found."
107 fi
108 printprocesslog "INFO getting information from DB for "$rawfile
109 # get information from DB
110 query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
111 runtype=`sendquery`
112 query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
113 tstart=`sendquery`
114 query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
115 tstop=`sendquery`
116 query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
117 tstart2=`sendquery`
118 query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
119 tstop2=`sendquery`
120 if [ "$tstart" = "" ] || [ "$tstop" = "" ] || [ "$runtype" = "n/a" ]
121 then
122 printprocesslog "WARN: "$rawfile" not available and one of the following information empty in DB: fRunStart("$tstart"), fRunStop("$tstop"), fRunTypeKey("$runtype") "
123 continue
124 fi
125 else
126 #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
127 #if [ "$checkfitsfile" == "" ]
128 #then
129 # numfitserrors=1
130 # printprocesslog "WARN: "$rawfile" probably corrupted."
131 # continue
132 #fi
133 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
134 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
135 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
136 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
137 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
138 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
139 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
140 then
141 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
142 continue
143 fi
144 # assuming that at least TSTARTI and TSTOPI are consistent
145 if [ $tstarti -gt 30000 ]
146 then
147 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
148 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
149 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
150 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
151 else
152 tstart=`echo " $tstarti + $tstartf " | bc -l`
153 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
154 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
155 tstop=`echo " $tstopi + $tstopf " | bc -l`
156 fi
157 #echo `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
158 fi
159
160 # build query to update runinfo in DB
161 query="UPDATE RunInfo SET "
162
163 # get information from weather: T
164 if [ -e $magicweatherfile ] && [ $weathernumerrors -eq 0 ]
165 then
166 #echo $magicweatherfile" "$tstart" "$tstop
167 printprocesslog "DEBUG root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,"$tstart"\,"$tstop"\)"
168 mtemps=( `root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
169 check1=$?
170 case $check1 in
171 0) printprocesslog "INFO got info from "$magicweatherfile
172 ;;
173 *) printprocesslog "WARN getting info from "$magicweatherfile" failed (check1="$check1")."
174 ;;
175 esac
176 #echo ${mtemps[@]}
177 if [ "${mtemps[0]}" == "" ]
178 then
179 query=$query"fOutsideTempMean=NULL"
180 else
181 query=$query"fOutsideTempMean="${mtemps[0]}
182 fi
183 if [ "${mtemps[1]}" == "" ]
184 then
185 query=$query", fOutsideTempRms=NULL"
186 else
187 query=$query", fOutsideTempRms="${mtemps[1]}
188 fi
189 if [ "${mtemps[2]}" == "" ]
190 then
191 query=$query", fHumidityMean=NULL"
192 else
193 query=$query", fHumidityMean="${mtemps[2]}
194 fi
195 if [ "${mtemps[3]}" == "" ]
196 then
197 query=$query", fHumidityRms=NULL"
198 else
199 query=$query", fHumidityRms="${mtemps[3]}
200 fi
201 if [ "${mtemps[4]}" == "" ]
202 then
203 query=$query", fAirPressureMean=NULL"
204 else
205 query=$query", fAirPressureMean="${mtemps[4]}
206 fi
207 if [ "${mtemps[5]}" == "" ]
208 then
209 query=$query", fAirPressureRms=NULL"
210 else
211 query=$query", fAirPressureRms="${mtemps[5]}
212 fi
213 if [ "${mtemps[6]}" == "" ]
214 then
215 query=$query", fDewPointMean=NULL"
216 else
217 query=$query", fDewPointMean="${mtemps[6]}
218 fi
219 if [ "${mtemps[7]}" == "" ]
220 then
221 query=$query", fDewPointRms=NULL"
222 else
223 query=$query", fDewPointRms="${mtemps[7]}
224 fi
225 if [ "${mtemps[8]}" == "" ]
226 then
227 query=$query", fWindSpeedMean=NULL"
228 else
229 query=$query", fWindSpeedMean="${mtemps[8]}
230 fi
231 if [ "${mtemps[9]}" == "" ]
232 then
233 query=$query", fWindSpeedRms=NULL"
234 else
235 query=$query", fWindSpeedRms="${mtemps[9]}
236 fi
237 if [ "${mtemps[10]}" == "" ]
238 then
239 query=$query", fWindGustMean=NULL"
240 else
241 query=$query", fWindGustMean="${mtemps[10]}
242 fi
243 if [ "${mtemps[119]}" == "" ]
244 then
245 query=$query", fWindGustRms=NULL"
246 else
247 query=$query", fWindGustRms="${mtemps[11]}
248 fi
249 else
250 query=$query"fOutsideTempMean=NULL"
251 query=$query", fOutsideTempRms=NULL"
252 query=$query", fHumidityMean=NULL"
253 query=$query", fHumidityRms=NULL"
254 query=$query", fAirPressureMean=NULL"
255 query=$query", fAirPressureRms=NULL"
256 query=$query", fDewPointMean=NULL"
257 query=$query", fDewPointRms=NULL"
258 query=$query", fWindSpeedMean=NULL"
259 query=$query", fWindSpeedRms=NULL"
260 query=$query", fWindGustMean=NULL"
261 query=$query", fWindGustRms=NULL"
262 fi
263
264 # add where condition
265 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
266
267 # send query to DB
268 sendquery >/dev/null
269 done
270done
271
272finish
273
274
Note: See TracBrowser for help on using the repository browser.