source: trunk/DataCheck/Processing/FillAuxMagic.sh@ 19007

Last change on this file since 19007 was 18944, checked in by Daniela Dorner, 7 years ago
added new values
  • Property svn:executable set to *
File size: 8.4 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillAuxTemp-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30 getdates $certaindate
31 echo `date`": processing the following night(s): "${dates[@]}
32else
33 # get all night
34 #getdates "all"
35 # get last 6 nights
36 getdates 6
37fi
38
39
40
41printprocesslog "INFO processing the following night(s): "${dates[@]}
42echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
43
44cd $mars
45
46# do filling of aux data
47for date in ${dates[@]}
48do
49 auxdir=$auxdata/$date
50 runnumber=`echo $date | sed -e 's/\///g'`
51 if [ $runnumber -lt 20120328 ]
52 then
53 continue
54 fi
55
56 # check if aux files are available from that night
57 if ! [ -d $auxdir ]
58 then
59 printprocesslog "INFO no data available in "$auxdir
60 continue
61 else
62 printprocesslog "INFO processing files in "$auxdir
63 fi
64
65 # get file numbers from DB
66 # but only for not-corrupted files
67 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
68 if [ "$doupdate" = "no" ]
69 then
70 query=$query" AND ISNULL(fCameraTempMean) "
71 fi
72 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
73 filenumbers=( `sendquery $query` )
74 # proceed only if there are files available
75 if [ ${#filenumbers} -eq 0 ]
76 then
77 printprocesslog "INFO No files found in the DB for night "$date
78 continue
79 fi
80
81 magicweatherfile=$auxdir/$runnumber.MAGIC_WEATHER_DATA.fits
82 if ! [ -e $magicweatherfile ]
83 then
84 printprocesslog "WARN "$magicweatherfile" not found."
85 else
86 weathernumerrors=`fverify $magicweatherfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
87 if [ $weathernumerrors -gt 0 ]
88 then
89 printprocesslog "WARN for $magicweatherfile fverify returned "$weathernumerrors" error(s)."
90 fi
91 fi
92
93 # fill auxiliary information for files
94 for filenum in ${filenumbers[@]}
95 do
96 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
97 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
98 # get information from rawfile
99 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
100 if ! [ -e $rawfile ]
101 then
102 if [ $runnumber -lt $checknight ]
103 then
104 printprocesslog "WARN "$rawfile" not found."
105 else
106 printprocesslog "INFO "$rawfile" not found."
107 fi
108 continue
109 fi
110 #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
111 #if [ "$checkfitsfile" == "" ]
112 #then
113 # numfitserrors=1
114 # printprocesslog "WARN: "$rawfile" probably corrupted."
115 # continue
116 #fi
117 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
118 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
119 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
120 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
121 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
122 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
123 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
124 then
125 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
126 continue
127 fi
128 # assuming that at least TSTARTI and TSTOPI are consistent
129 if [ $tstarti -gt 30000 ]
130 then
131 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
132 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
133 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
134 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
135 else
136 tstart=`echo " $tstarti + $tstartf " | bc -l`
137 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
138 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
139 tstop=`echo " $tstopi + $tstopf " | bc -l`
140 fi
141 #echo `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
142
143 # build query to update runinfo in DB
144 query="UPDATE RunInfo SET "
145
146 # get information from weather: T
147 if [ -e $magicweatherfile ] && [ $weathernumerrors -eq 0 ]
148 then
149 #echo $magicweatherfile" "$tstart" "$tstop
150 #root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result"
151 mtemps=( `root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
152 #echo ${mtemps[@]}
153 if [ "${mtemps[0]}" == "" ]
154 then
155 query=$query"fOutsideTempMean=NULL"
156 else
157 query=$query"fOutsideTempMean="${mtemps[0]}
158 fi
159 if [ "${mtemps[1]}" == "" ]
160 then
161 query=$query", fOutsideTempRms=NULL"
162 else
163 query=$query", fOutsideTempRms="${mtemps[1]}
164 fi
165 if [ "${mtemps[2]}" == "" ]
166 then
167 query=$query", fHumidityMean=NULL"
168 else
169 query=$query", fHumidityMean="${mtemps[2]}
170 fi
171 if [ "${mtemps[3]}" == "" ]
172 then
173 query=$query", fHumidityRms=NULL"
174 else
175 query=$query", fHumidityRms="${mtemps[3]}
176 fi
177 if [ "${mtemps[4]}" == "" ]
178 then
179 query=$query", fAirPressureMean=NULL"
180 else
181 query=$query", fAirPressureMean="${mtemps[4]}
182 fi
183 if [ "${mtemps[5]}" == "" ]
184 then
185 query=$query", fAirPressureRms=NULL"
186 else
187 query=$query", fAirPressureRms="${mtemps[5]}
188 fi
189 if [ "${mtemps[6]}" == "" ]
190 then
191 query=$query", fDewPointMean=NULL"
192 else
193 query=$query", fDewPointMean="${mtemps[6]}
194 fi
195 if [ "${mtemps[7]}" == "" ]
196 then
197 query=$query", fDewPointRms=NULL"
198 else
199 query=$query", fDewPointRms="${mtemps[7]}
200 fi
201 if [ "${mtemps[8]}" == "" ]
202 then
203 query=$query", fWindSpeedMean=NULL"
204 else
205 query=$query", fWindSpeedMean="${mtemps[8]}
206 fi
207 if [ "${mtemps[9]}" == "" ]
208 then
209 query=$query", fWindSpeedRms=NULL"
210 else
211 query=$query", fWindSpeedRms="${mtemps[9]}
212 fi
213 if [ "${mtemps[10]}" == "" ]
214 then
215 query=$query", fWindGustMean=NULL"
216 else
217 query=$query", fWindGustMean="${mtemps[10]}
218 fi
219 if [ "${mtemps[119]}" == "" ]
220 then
221 query=$query", fWindGustRms=NULL"
222 else
223 query=$query", fWindGustRms="${mtemps[11]}
224 fi
225 else
226 query=$query"fOutsideTempMean=NULL"
227 query=$query", fOutsideTempRms=NULL"
228 query=$query", fHumidityMean=NULL"
229 query=$query", fHumidityRms=NULL"
230 query=$query", fAirPressureMean=NULL"
231 query=$query", fAirPressureRms=NULL"
232 query=$query", fDewPointMean=NULL"
233 query=$query", fDewPointRms=NULL"
234 query=$query", fWindSpeedMean=NULL"
235 query=$query", fWindSpeedRms=NULL"
236 query=$query", fWindGustMean=NULL"
237 query=$query", fWindGustRms=NULL"
238 fi
239
240 # add where condition
241 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
242
243 # send query to DB
244 sendquery >/dev/null
245 done
246done
247
248finish
249
250
Note: See TracBrowser for help on using the repository browser.