source: trunk/DataCheck/Processing/FillAuxMagic.sh @ 19484

Last change on this file since 19484 was 19484, checked in by dorner, 4 months ago
added check for cases where both the rawfile is not in the archive and the information not in the DB
  • Property svn:executable set to *
File size: 9.9 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7   doupdate="yes" # update all entries (needed when new fields have been added)
8   doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillAuxTemp-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then 
23   printprocesslog "ERROR "$factpath"/fitsdump is not available."
24   finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30   getdates $certaindate
31   echo  `date`": processing the following night(s): "${dates[@]} 
32else
33   # get all night
34   #getdates "all"
35   # get last 6 nights
36   getdates 6
37fi
38
39
40
41printprocesslog "INFO processing the following night(s): "${dates[@]}
42echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
43
44cd $mars
45
46# do filling of aux data
47for date in ${dates[@]}
48do 
49   auxdir=$auxdata/$date
50   runnumber=`echo $date | sed -e 's/\///g'`
51   if [ $runnumber -lt 20120328 ]
52   then
53      continue
54   fi
55
56   # check if aux files are available from that night
57   if ! [ -d $auxdir ]
58   then
59      printprocesslog "INFO no data available in "$auxdir
60      continue
61   else
62      printprocesslog "INFO processing files in "$auxdir
63   fi
64
65   # get file numbers from DB
66   #   but only for not-corrupted files
67   query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
68   if [ "$doupdate" = "no" ]
69   then
70      query=$query" AND ISNULL(fCameraTempMean) "
71   fi
72   printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
73   filenumbers=( `sendquery $query` )
74   # proceed only if there are files available
75   if [ ${#filenumbers} -eq 0 ]
76   then
77      printprocesslog "INFO No files found in the DB for night "$date
78      continue
79   fi
80
81   magicweatherfile=$auxdir/$runnumber.MAGIC_WEATHER_DATA.fits
82   if ! [ -e $magicweatherfile ]
83   then 
84      printprocesslog "WARN "$magicweatherfile" not found."
85   else
86      weathernumerrors=`fverify $magicweatherfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
87      if [ $weathernumerrors -gt 0 ]
88      then 
89         printprocesslog "WARN for $magicweatherfile fverify returned "$weathernumerrors" error(s)."
90      fi
91   fi
92
93   # fill auxiliary information for files
94   for filenum in ${filenumbers[@]}
95   do
96      printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
97      echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
98      # get information from rawfile
99      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
100      if ! [ -e $rawfile ]
101      then 
102         if [ $runnumber -lt $checknight ] 
103         then 
104            printprocesslog "WARN "$rawfile" not found."
105         else
106            printprocesslog "INFO "$rawfile" not found."
107         fi
108         printprocesslog "INFO getting information from DB for "$rawfile
109         # get information from DB
110         query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
111         runtype=`sendquery`
112         query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
113         tstart=`sendquery`
114         query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
115         tstop=`sendquery`
116         query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
117         tstart2=`sendquery`
118         query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
119         tstop2=`sendquery`
120         if [ "$tstart" = "" ] || [ "$tstop" = "" ] || [ "$runtype" = "n/a" ]
121         then 
122            printprocesslog "WARN: "$rawfile" not available and one of the following information empty in DB: fRunStart("$tstart"), fRunStop("$tstop"), fRunTypeKey("$runtype") "
123            continue
124         fi
125      else
126         #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
127         #if [ "$checkfitsfile" == "" ]
128         #then
129         #   numfitserrors=1
130         #   printprocesslog "WARN: "$rawfile" probably corrupted."
131         #   continue
132         #fi
133         runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
134         mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
135         tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
136         tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
137         tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
138         tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
139         if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
140         then
141            printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
142            continue
143         fi
144         # assuming that at least TSTARTI and TSTOPI are consistent
145         if [ $tstarti -gt 30000 ]
146         then
147            tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
148            tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
149            #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
150            tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
151         else
152            tstart=`echo " $tstarti + $tstartf " | bc -l`
153            tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
154            #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
155            tstop=`echo " $tstopi + $tstopf " | bc -l`
156         fi
157         #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
158      fi
159
160      # build query to update runinfo in DB
161      query="UPDATE RunInfo SET "
162
163      # get information from weather: T
164      if [ -e $magicweatherfile ] && [ $weathernumerrors -eq 0 ]
165      then 
166         #echo $magicweatherfile" "$tstart" "$tstop
167         printprocesslog "DEBUG root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,"$tstart"\,"$tstop"\)"
168         mtemps=( `root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
169         check1=$?
170         case $check1 in
171            0)   printprocesslog "INFO got info from "$magicweatherfile
172                 ;;
173            *)   printprocesslog "WARN getting info from "$magicweatherfile" failed (check1="$check1")."
174                 ;;
175         esac
176         #echo ${mtemps[@]}
177         if [ "${mtemps[0]}" == "" ]
178         then 
179            query=$query"fOutsideTempMean=NULL"
180         else
181            query=$query"fOutsideTempMean="${mtemps[0]}
182         fi
183         if [ "${mtemps[1]}" == "" ]
184         then 
185            query=$query", fOutsideTempRms=NULL"
186         else
187            query=$query", fOutsideTempRms="${mtemps[1]}
188         fi
189         if [ "${mtemps[2]}" == "" ]
190         then 
191            query=$query", fHumidityMean=NULL"
192         else
193            query=$query", fHumidityMean="${mtemps[2]}
194         fi
195         if [ "${mtemps[3]}" == "" ]
196         then 
197            query=$query", fHumidityRms=NULL"
198         else
199            query=$query", fHumidityRms="${mtemps[3]}
200         fi
201         if [ "${mtemps[4]}" == "" ]
202         then 
203            query=$query", fAirPressureMean=NULL"
204         else
205            query=$query", fAirPressureMean="${mtemps[4]}
206         fi
207         if [ "${mtemps[5]}" == "" ]
208         then 
209            query=$query", fAirPressureRms=NULL"
210         else
211            query=$query", fAirPressureRms="${mtemps[5]}
212         fi
213         if [ "${mtemps[6]}" == "" ]
214         then 
215            query=$query", fDewPointMean=NULL"
216         else
217            query=$query", fDewPointMean="${mtemps[6]}
218         fi
219         if [ "${mtemps[7]}" == "" ]
220         then 
221            query=$query", fDewPointRms=NULL"
222         else
223            query=$query", fDewPointRms="${mtemps[7]}
224         fi
225         if [ "${mtemps[8]}" == "" ]
226         then 
227            query=$query", fWindSpeedMean=NULL"
228         else
229            query=$query", fWindSpeedMean="${mtemps[8]}
230         fi
231         if [ "${mtemps[9]}" == "" ]
232         then 
233            query=$query", fWindSpeedRms=NULL"
234         else
235            query=$query", fWindSpeedRms="${mtemps[9]}
236         fi
237         if [ "${mtemps[10]}" == "" ]
238         then 
239            query=$query", fWindGustMean=NULL"
240         else
241            query=$query", fWindGustMean="${mtemps[10]}
242         fi
243         if [ "${mtemps[119]}" == "" ]
244         then 
245            query=$query", fWindGustRms=NULL"
246         else
247            query=$query", fWindGustRms="${mtemps[11]}
248         fi
249      else
250         query=$query"fOutsideTempMean=NULL"
251         query=$query", fOutsideTempRms=NULL"
252         query=$query", fHumidityMean=NULL"
253         query=$query", fHumidityRms=NULL"
254         query=$query", fAirPressureMean=NULL"
255         query=$query", fAirPressureRms=NULL"
256         query=$query", fDewPointMean=NULL"
257         query=$query", fDewPointRms=NULL"
258         query=$query", fWindSpeedMean=NULL"
259         query=$query", fWindSpeedRms=NULL"
260         query=$query", fWindGustMean=NULL"
261         query=$query", fWindGustRms=NULL"
262      fi
263
264      # add where condition
265      query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
266     
267      # send query to DB
268      sendquery >/dev/null
269   done
270done
271
272finish
273
274
Note: See TracBrowser for help on using the repository browser.