source: trunk/DataCheck/Processing/FillAuxMagic.sh@ 19440

Last change on this file since 19440 was 19201, checked in by Daniela Dorner, 6 years ago
usage of check_file_avail function
  • Property svn:executable set to *
File size: 9.3 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillAuxTemp-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 getdates $certaindate
31 echo `date`": processing the following night(s): "${dates[@]}
32else
33 # get all night
34 #getdates "all"
35 # get last 6 nights
36 getdates 6
37fi
38
39
40
41printprocesslog "INFO processing the following night(s): "${dates[@]}
42echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
43
44cd $mars
45
46# do filling of aux data
47for date in ${dates[@]}
48do
49 auxdir=$auxdata/$date
50 runnumber=`echo $date | sed -e 's/\///g'`
51 if [ $runnumber -lt 20120328 ]
52 then
53 continue
54 fi
55
56 # check if aux files are available from that night
57 if ! [ -d $auxdir ]
58 then
59 printprocesslog "INFO no data available in "$auxdir
60 continue
61 else
62 printprocesslog "INFO processing files in "$auxdir
63 fi
64
65 # get file numbers from DB
66 # but only for not-corrupted files
67 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
68 if [ "$doupdate" = "no" ]
69 then
70 query=$query" AND ISNULL(fCameraTempMean) "
71 fi
72 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
73 filenumbers=( `sendquery $query` )
74 # proceed only if there are files available
75 if [ ${#filenumbers} -eq 0 ]
76 then
77 printprocesslog "INFO No files found in the DB for night "$date
78 continue
79 fi
80
81 magicweatherfile=$auxdir/$runnumber.MAGIC_WEATHER_DATA.fits
82 if ! [ -e $magicweatherfile ]
83 then
84 printprocesslog "WARN "$magicweatherfile" not found."
85 else
86 weathernumerrors=`fverify $magicweatherfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
87 if [ $weathernumerrors -gt 0 ]
88 then
89 printprocesslog "WARN for $magicweatherfile fverify returned "$weathernumerrors" error(s)."
90 fi
91 fi
92
93 # fill auxiliary information for files
94 for filenum in ${filenumbers[@]}
95 do
96 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
97 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
98 # get information from rawfile
99 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
100 if ! [ -e $rawfile ]
101 then
102 if [ $runnumber -lt $checknight ]
103 then
104 printprocesslog "WARN "$rawfile" not found."
105 else
106 printprocesslog "INFO "$rawfile" not found."
107 fi
108 printprocesslog "INFO getting information from DB for "$rawfile
109 # get information from DB
110 query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
111 runtype=`sendquery`
112 query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
113 tstart=`sendquery`
114 query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
115 tstop=`sendquery`
116 query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
117 tstart2=`sendquery`
118 query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
119 tstop2=`sendquery`
120 else
121 #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
122 #if [ "$checkfitsfile" == "" ]
123 #then
124 # numfitserrors=1
125 # printprocesslog "WARN: "$rawfile" probably corrupted."
126 # continue
127 #fi
128 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
129 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
130 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
131 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
132 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
133 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
134 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
135 then
136 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
137 continue
138 fi
139 # assuming that at least TSTARTI and TSTOPI are consistent
140 if [ $tstarti -gt 30000 ]
141 then
142 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
143 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
144 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
145 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
146 else
147 tstart=`echo " $tstarti + $tstartf " | bc -l`
148 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
149 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
150 tstop=`echo " $tstopi + $tstopf " | bc -l`
151 fi
152 #echo `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
153 fi
154
155 # build query to update runinfo in DB
156 query="UPDATE RunInfo SET "
157
158 # get information from weather: T
159 if [ -e $magicweatherfile ] && [ $weathernumerrors -eq 0 ]
160 then
161 #echo $magicweatherfile" "$tstart" "$tstop
162 #root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result"
163 mtemps=( `root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
164 #echo ${mtemps[@]}
165 if [ "${mtemps[0]}" == "" ]
166 then
167 query=$query"fOutsideTempMean=NULL"
168 else
169 query=$query"fOutsideTempMean="${mtemps[0]}
170 fi
171 if [ "${mtemps[1]}" == "" ]
172 then
173 query=$query", fOutsideTempRms=NULL"
174 else
175 query=$query", fOutsideTempRms="${mtemps[1]}
176 fi
177 if [ "${mtemps[2]}" == "" ]
178 then
179 query=$query", fHumidityMean=NULL"
180 else
181 query=$query", fHumidityMean="${mtemps[2]}
182 fi
183 if [ "${mtemps[3]}" == "" ]
184 then
185 query=$query", fHumidityRms=NULL"
186 else
187 query=$query", fHumidityRms="${mtemps[3]}
188 fi
189 if [ "${mtemps[4]}" == "" ]
190 then
191 query=$query", fAirPressureMean=NULL"
192 else
193 query=$query", fAirPressureMean="${mtemps[4]}
194 fi
195 if [ "${mtemps[5]}" == "" ]
196 then
197 query=$query", fAirPressureRms=NULL"
198 else
199 query=$query", fAirPressureRms="${mtemps[5]}
200 fi
201 if [ "${mtemps[6]}" == "" ]
202 then
203 query=$query", fDewPointMean=NULL"
204 else
205 query=$query", fDewPointMean="${mtemps[6]}
206 fi
207 if [ "${mtemps[7]}" == "" ]
208 then
209 query=$query", fDewPointRms=NULL"
210 else
211 query=$query", fDewPointRms="${mtemps[7]}
212 fi
213 if [ "${mtemps[8]}" == "" ]
214 then
215 query=$query", fWindSpeedMean=NULL"
216 else
217 query=$query", fWindSpeedMean="${mtemps[8]}
218 fi
219 if [ "${mtemps[9]}" == "" ]
220 then
221 query=$query", fWindSpeedRms=NULL"
222 else
223 query=$query", fWindSpeedRms="${mtemps[9]}
224 fi
225 if [ "${mtemps[10]}" == "" ]
226 then
227 query=$query", fWindGustMean=NULL"
228 else
229 query=$query", fWindGustMean="${mtemps[10]}
230 fi
231 if [ "${mtemps[119]}" == "" ]
232 then
233 query=$query", fWindGustRms=NULL"
234 else
235 query=$query", fWindGustRms="${mtemps[11]}
236 fi
237 else
238 query=$query"fOutsideTempMean=NULL"
239 query=$query", fOutsideTempRms=NULL"
240 query=$query", fHumidityMean=NULL"
241 query=$query", fHumidityRms=NULL"
242 query=$query", fAirPressureMean=NULL"
243 query=$query", fAirPressureRms=NULL"
244 query=$query", fDewPointMean=NULL"
245 query=$query", fDewPointRms=NULL"
246 query=$query", fWindSpeedMean=NULL"
247 query=$query", fWindSpeedRms=NULL"
248 query=$query", fWindGustMean=NULL"
249 query=$query", fWindGustRms=NULL"
250 fi
251
252 # add where condition
253 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
254
255 # send query to DB
256 sendquery >/dev/null
257 done
258done
259
260finish
261
262
Note: See TracBrowser for help on using the repository browser.