source: trunk/DataCheck/Processing/FillAuxMagic.sh @ 19201

Last change on this file since 19201 was 19201, checked in by dorner, 9 months ago
usage of check_file_avail function
  • Property svn:executable set to *
File size: 9.3 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7   doupdate="yes" # update all entries (needed when new fields have been added)
8   doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillAuxTemp-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then 
23   printprocesslog "ERROR "$factpath"/fitsdump is not available."
24   finish
25fi
26
27# get dates
28if [ "$certaindate" != "" ]
29then
30   getdates $certaindate
31   echo  `date`": processing the following night(s): "${dates[@]} 
32else
33   # get all night
34   #getdates "all"
35   # get last 6 nights
36   getdates 6
37fi
38
39
40
41printprocesslog "INFO processing the following night(s): "${dates[@]}
42echo  `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
43
44cd $mars
45
46# do filling of aux data
47for date in ${dates[@]}
48do 
49   auxdir=$auxdata/$date
50   runnumber=`echo $date | sed -e 's/\///g'`
51   if [ $runnumber -lt 20120328 ]
52   then
53      continue
54   fi
55
56   # check if aux files are available from that night
57   if ! [ -d $auxdir ]
58   then
59      printprocesslog "INFO no data available in "$auxdir
60      continue
61   else
62      printprocesslog "INFO processing files in "$auxdir
63   fi
64
65   # get file numbers from DB
66   #   but only for not-corrupted files
67   query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
68   if [ "$doupdate" = "no" ]
69   then
70      query=$query" AND ISNULL(fCameraTempMean) "
71   fi
72   printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
73   filenumbers=( `sendquery $query` )
74   # proceed only if there are files available
75   if [ ${#filenumbers} -eq 0 ]
76   then
77      printprocesslog "INFO No files found in the DB for night "$date
78      continue
79   fi
80
81   magicweatherfile=$auxdir/$runnumber.MAGIC_WEATHER_DATA.fits
82   if ! [ -e $magicweatherfile ]
83   then 
84      printprocesslog "WARN "$magicweatherfile" not found."
85   else
86      weathernumerrors=`fverify $magicweatherfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
87      if [ $weathernumerrors -gt 0 ]
88      then 
89         printprocesslog "WARN for $magicweatherfile fverify returned "$weathernumerrors" error(s)."
90      fi
91   fi
92
93   # fill auxiliary information for files
94   for filenum in ${filenumbers[@]}
95   do
96      printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
97      echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
98      # get information from rawfile
99      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
100      if ! [ -e $rawfile ]
101      then 
102         if [ $runnumber -lt $checknight ] 
103         then 
104            printprocesslog "WARN "$rawfile" not found."
105         else
106            printprocesslog "INFO "$rawfile" not found."
107         fi
108         printprocesslog "INFO getting information from DB for "$rawfile
109         # get information from DB
110         query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
111         runtype=`sendquery`
112         query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
113         tstart=`sendquery`
114         query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
115         tstop=`sendquery`
116         query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
117         tstart2=`sendquery`
118         query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
119         tstop2=`sendquery`
120      else
121         #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
122         #if [ "$checkfitsfile" == "" ]
123         #then
124         #   numfitserrors=1
125         #   printprocesslog "WARN: "$rawfile" probably corrupted."
126         #   continue
127         #fi
128         runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
129         mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
130         tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
131         tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
132         tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
133         tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
134         if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
135         then
136            printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
137            continue
138         fi
139         # assuming that at least TSTARTI and TSTOPI are consistent
140         if [ $tstarti -gt 30000 ]
141         then
142            tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
143            tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
144            #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
145            tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
146         else
147            tstart=`echo " $tstarti + $tstartf " | bc -l`
148            tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
149            #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
150            tstop=`echo " $tstopi + $tstopf " | bc -l`
151         fi
152         #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
153      fi
154
155      # build query to update runinfo in DB
156      query="UPDATE RunInfo SET "
157
158      # get information from weather: T
159      if [ -e $magicweatherfile ] && [ $weathernumerrors -eq 0 ]
160      then 
161         #echo $magicweatherfile" "$tstart" "$tstop
162         #root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result"
163         mtemps=( `root -q -b -l fact/processing/magicweather.C\("\"$magicweatherfile\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` )
164         #echo ${mtemps[@]}
165         if [ "${mtemps[0]}" == "" ]
166         then 
167            query=$query"fOutsideTempMean=NULL"
168         else
169            query=$query"fOutsideTempMean="${mtemps[0]}
170         fi
171         if [ "${mtemps[1]}" == "" ]
172         then 
173            query=$query", fOutsideTempRms=NULL"
174         else
175            query=$query", fOutsideTempRms="${mtemps[1]}
176         fi
177         if [ "${mtemps[2]}" == "" ]
178         then 
179            query=$query", fHumidityMean=NULL"
180         else
181            query=$query", fHumidityMean="${mtemps[2]}
182         fi
183         if [ "${mtemps[3]}" == "" ]
184         then 
185            query=$query", fHumidityRms=NULL"
186         else
187            query=$query", fHumidityRms="${mtemps[3]}
188         fi
189         if [ "${mtemps[4]}" == "" ]
190         then 
191            query=$query", fAirPressureMean=NULL"
192         else
193            query=$query", fAirPressureMean="${mtemps[4]}
194         fi
195         if [ "${mtemps[5]}" == "" ]
196         then 
197            query=$query", fAirPressureRms=NULL"
198         else
199            query=$query", fAirPressureRms="${mtemps[5]}
200         fi
201         if [ "${mtemps[6]}" == "" ]
202         then 
203            query=$query", fDewPointMean=NULL"
204         else
205            query=$query", fDewPointMean="${mtemps[6]}
206         fi
207         if [ "${mtemps[7]}" == "" ]
208         then 
209            query=$query", fDewPointRms=NULL"
210         else
211            query=$query", fDewPointRms="${mtemps[7]}
212         fi
213         if [ "${mtemps[8]}" == "" ]
214         then 
215            query=$query", fWindSpeedMean=NULL"
216         else
217            query=$query", fWindSpeedMean="${mtemps[8]}
218         fi
219         if [ "${mtemps[9]}" == "" ]
220         then 
221            query=$query", fWindSpeedRms=NULL"
222         else
223            query=$query", fWindSpeedRms="${mtemps[9]}
224         fi
225         if [ "${mtemps[10]}" == "" ]
226         then 
227            query=$query", fWindGustMean=NULL"
228         else
229            query=$query", fWindGustMean="${mtemps[10]}
230         fi
231         if [ "${mtemps[119]}" == "" ]
232         then 
233            query=$query", fWindGustRms=NULL"
234         else
235            query=$query", fWindGustRms="${mtemps[11]}
236         fi
237      else
238         query=$query"fOutsideTempMean=NULL"
239         query=$query", fOutsideTempRms=NULL"
240         query=$query", fHumidityMean=NULL"
241         query=$query", fHumidityRms=NULL"
242         query=$query", fAirPressureMean=NULL"
243         query=$query", fAirPressureRms=NULL"
244         query=$query", fDewPointMean=NULL"
245         query=$query", fDewPointRms=NULL"
246         query=$query", fWindSpeedMean=NULL"
247         query=$query", fWindSpeedRms=NULL"
248         query=$query", fWindGustMean=NULL"
249         query=$query", fWindGustRms=NULL"
250      fi
251
252      # add where condition
253      query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
254     
255      # send query to DB
256      sendquery >/dev/null
257   done
258done
259
260finish
261
262
Note: See TracBrowser for help on using the repository browser.