source: trunk/DataCheck/Processing/FillAuxMagic.sh @ 19484

Last change on this file since 19484 was 19484, checked in by dorner, 14 months ago
added check for cases where both the rawfile is not in the archive and the information not in the DB
  • Property svn:executable set to *
File size: 9.9 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7   doupdate="yes" # update all entries (needed when new fields have been added)
8   doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillAuxTemp-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then 
23   printprocesslog "ERROR "$factpath"/fitsdump is not available."
24   finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30   getdates $certaindate
31   echo  `date`": processing the following night(s): "${dates[@]} 
32else
33   # get all night
34   #getdates "all"
35   # get last 6 nights
36   getdates 6
37fi
38
39
40
41printprocesslog "INFO processing the following night(s): "${dates[@]}
42echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
43
44cd $mars
45
46# do filling of aux data
47for date in ${dates[@]}
48do 
49   auxdir=$auxdata/$date
50   runnumber=`echo $date | sed -e 's/\///g'`
51   if [ $runnumber -lt 20120328 ]
52   then
53      continue
54   fi
55
56   # check if aux files are available from that night
57   if ! [ -d $auxdir ]
58   then
59      printprocesslog "INFO no data available in "$auxdir
60      continue
61   else
62      printprocesslog "INFO processing files in "$auxdir
63   fi
64
65   # get file numbers from DB
66   #   but only for not-corrupted files
67   query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
68   if [ "$doupdate" = "no" ]
69   then
70      query=$query" AND ISNULL(fCameraTempMean) "
71   fi
72   printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
73   filenumbers=( `sendquery $query` )
74   # proceed only if there are files available
75   if [ ${#filenumbers} -eq 0 ]
76   then
77      printprocesslog "INFO No files found in the DB for night "$date
78      continue
79   fi
80
81   magicweatherfile=$auxdir/$runnumber.MAGIC_WEATHER_DATA.fits
82   if ! [ -e $magicweatherfile ]
83   then 
84      printprocesslog "WARN "$magicweatherfile" not found."
85   else
86      weathernumerrors=`fverify $magicweatherfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
87      if [ $weathernumerrors -gt 0 ]
88      then 
89         printprocesslog "WARN for $magicweatherfile fverify returned "$weathernumerrors" error(s)."
90      fi
91   fi
92
93   # fill auxiliary information for files
94   for filenum in ${filenumbers[@]}
95   do
96      printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
97      echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
98      # get information from rawfile
99      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
100      if ! [ -e $rawfile ]
101      then 
102         if [ $runnumber -lt $checknight ] 
103         then 
104            printprocesslog "WARN "$rawfile" not found."
105         else
106            printprocesslog "INFO "$rawfile" not found."
107         fi
108         printprocesslog "INFO getting information from DB for "$rawfile
109         # get information from DB
110         query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
111         runtype=`sendquery`
112         query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
113         tstart=`sendquery`
114         query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
115         tstop=`sendquery`
116         query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
117         tstart2=`sendquery`
118         query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
119         tstop2=`sendquery`
120         if [ "$tstart" = "" ] || [ "$tstop" = "" ] || [ "$runtype" = "n/a" ]
121         then 
122            printprocesslog "WARN: "$rawfile" not available and one of the following information empty in DB: fRunStart("$tstart"), fRunStop("$tstop"), fRunTypeKey("$runtype") "
123            continue
124         fi
125      else
126         #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
127         #if [ "$checkfitsfile" == "" ]
128         #then
129         #   numfitserrors=1
130         #   printprocesslog "WARN: "$rawfile" probably corrupted."
131         #   continue
132         #fi
133         runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
134         mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
135         tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
136         tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
137         tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
138         tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
139         if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
140         then
141            printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
142            continue
143         fi
144         # assuming that at least TSTARTI and TSTOPI are consistent
145         if [ $tstarti -gt 30000 ]
146         then
147            tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
148            tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
149            #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
150            tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
151         else
152            tstart=`echo " $tstarti + $tstartf " | bc -l`
153            tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
154            #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
155            tstop=`echo " $tstopi + $tstopf " | bc -l`
156         fi
157         #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
158      fi
159
160      # build query to update runinfo in DB
161      query="UPDATE RunInfo SET "
162
163      # get information from weather: T
164      if [ -e $magicweatherfile ] && [ $weathernumerrors -eq 0 ]
165      then 
166         #echo $magicweatherfile" "$tstart" "$tstop
167         printprocesslog "DEBUG root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,"$tstart"\,"$tstop"\)"
168         mtemps=( `root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
169         check1=$?
170         case $check1 in
171            0)   printprocesslog "INFO got info from "$magicweatherfile
172                 ;;
173            *)   printprocesslog "WARN getting info from "$magicweatherfile" failed (check1="$check1")."
174                 ;;
175         esac
176         #echo ${mtemps[@]}
177         if [ "${mtemps[0]}" == "" ]
178         then 
179            query=$query"fOutsideTempMean=NULL"
180         else
181            query=$query"fOutsideTempMean="${mtemps[0]}
182         fi
183         if [ "${mtemps[1]}" == "" ]
184         then 
185            query=$query", fOutsideTempRms=NULL"
186         else
187            query=$query", fOutsideTempRms="${mtemps[1]}
188         fi
189         if [ "${mtemps[2]}" == "" ]
190         then 
191            query=$query", fHumidityMean=NULL"
192         else
193            query=$query", fHumidityMean="${mtemps[2]}
194         fi
195         if [ "${mtemps[3]}" == "" ]
196         then 
197            query=$query", fHumidityRms=NULL"
198         else
199            query=$query", fHumidityRms="${mtemps[3]}
200         fi
201         if [ "${mtemps[4]}" == "" ]
202         then 
203            query=$query", fAirPressureMean=NULL"
204         else
205            query=$query", fAirPressureMean="${mtemps[4]}
206         fi
207         if [ "${mtemps[5]}" == "" ]
208         then 
209            query=$query", fAirPressureRms=NULL"
210         else
211            query=$query", fAirPressureRms="${mtemps[5]}
212         fi
213         if [ "${mtemps[6]}" == "" ]
214         then 
215            query=$query", fDewPointMean=NULL"
216         else
217            query=$query", fDewPointMean="${mtemps[6]}
218         fi
219         if [ "${mtemps[7]}" == "" ]
220         then 
221            query=$query", fDewPointRms=NULL"
222         else
223            query=$query", fDewPointRms="${mtemps[7]}
224         fi
225         if [ "${mtemps[8]}" == "" ]
226         then 
227            query=$query", fWindSpeedMean=NULL"
228         else
229            query=$query", fWindSpeedMean="${mtemps[8]}
230         fi
231         if [ "${mtemps[9]}" == "" ]
232         then 
233            query=$query", fWindSpeedRms=NULL"
234         else
235            query=$query", fWindSpeedRms="${mtemps[9]}
236         fi
237         if [ "${mtemps[10]}" == "" ]
238         then 
239            query=$query", fWindGustMean=NULL"
240         else
241            query=$query", fWindGustMean="${mtemps[10]}
242         fi
243         if [ "${mtemps[119]}" == "" ]
244         then 
245            query=$query", fWindGustRms=NULL"
246         else
247            query=$query", fWindGustRms="${mtemps[11]}
248         fi
249      else
250         query=$query"fOutsideTempMean=NULL"
251         query=$query", fOutsideTempRms=NULL"
252         query=$query", fHumidityMean=NULL"
253         query=$query", fHumidityRms=NULL"
254         query=$query", fAirPressureMean=NULL"
255         query=$query", fAirPressureRms=NULL"
256         query=$query", fDewPointMean=NULL"
257         query=$query", fDewPointRms=NULL"
258         query=$query", fWindSpeedMean=NULL"
259         query=$query", fWindSpeedRms=NULL"
260         query=$query", fWindGustMean=NULL"
261         query=$query", fWindGustRms=NULL"
262      fi
263
264      # add where condition
265      query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
266     
267      # send query to DB
268      sendquery >/dev/null
269   done
270done
271
272finish
273
274
Note: See TracBrowser for help on using the repository browser.