source: branches/trigger_burst_research/Processing/FillAuxThresholds.sh@ 19923

Last change on this file since 19923 was 17907, checked in by Daniela Dorner, 10 years ago
included and/or improved check of variable certaindate
  • Property svn:executable set to *
File size: 7.8 KB
Line 
1#!/bin/bash
2
3# option whether to fill all row or only those where information is missing
4# $doupdate might be given as environment variable
5if [ "$doupdate" = "" ]
6then
7 doupdate="yes" # update all entries (needed when new fields have been added)
8 doupdate="no" # fill only entries which are not yet existing (default)
9fi
10
11source `dirname $0`/../Sourcefile.sh
12printprocesslog "INFO starting $0 with option doupdate="$doupdate
13
14logfile=$runlogpath"/FillThresholds-"$datetime".log"
15date >> $logfile
16
17# setup to use ftools
18source $HEADAS/headas-init.sh
19
20# check if software is available
21if ! ls $factpath/fitsdump >/dev/null 2>&1
22then
23 printprocesslog "ERROR "$factpath"/fitsdump is not available."
24 finish
25fi
26
27
28# get dates
29if [ "$certaindate" != "" ]
30then
31 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
32 if [ "$checkstring" = "" ]
33 then
34 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
35 finish
36 fi
37 getdates $certaindate
38else
39 # get all night
40 #getdates "all"
41 # get last 6 nights
42 getdates 6
43fi
44
45printprocesslog "INFO processing the following night(s): "${dates[@]}
46echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1
47
48#echo ${dates[@]}
49
50cd $mars
51
52# do filling of aux data
53for date in ${dates[@]}
54do
55 auxdir=$auxdata/$date
56 runnumber=`echo $date | sed -e 's/\///g'`
57
58 # get file numbers from DB
59 # but only for not-corrupted files
60 query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
61 if [ "$doupdate" = "no" ]
62 then
63 query=$query" AND ISNULL(fThresholdMedMean) AND ISNULL(fThresholdMinSet) "
64 fi
65 printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
66 filenumbers=( `sendquery $query` )
67 # proceed only if there are files available
68 if [ ${#filenumbers} -eq 0 ]
69 then
70 printprocesslog "INFO No files found in the DB for night "$date
71 continue
72 fi
73
74 # check if aux files are available from that night
75 if ! [ -d $auxdir ]
76 then
77 printprocesslog "INFO no data available in "$auxdir
78 continue
79 else
80 printprocesslog "INFO processing files in "$auxdir
81 fi
82
83 thresholdfile=$auxdir/$runnumber.RATE_CONTROL_THRESHOLD.fits
84 printprocesslog "INFO processing "$thresholdfile
85 echo "INFO processing "$thresholdfile >> $logfile 2>&1
86 if ! [ -e $thresholdfile ]
87 then
88 printprocesslog "WARN "$thresholdfile" not found."
89 continue
90 else
91 threshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
92 if [ $threshnumerrors -gt 0 ]
93 then
94 printprocesslog "WARN for $thresholdfile fverify returned "$threshnumerrors" error(s)."
95 fi
96 fi
97
98 thresholdfile2=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
99 #ls $thresholdfile2
100 if ! [ -e $thresholdfile2 ]
101 then
102 printprocesslog "WARN "$thresholdfile2" not found."
103 continue
104 else
105 threshnumerrors2=`fverify $thresholdfile2 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
106 if [ $threshnumerrors2 -gt 0 ]
107 then
108 printprocesslog "WARN for $thresholdfile2 fverify returned "$threshnumerrors2" error(s)."
109 fi
110 fi
111
112 # fill auxiliary information for files
113 for filenum in ${filenumbers[@]}
114 do
115 printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
116 echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1
117 # get information from rawfile
118 rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
119 if ! [ -e $rawfile ]
120 then
121 printprocesslog "ERROR: "$rawfile" not found."
122 continue
123 fi
124 runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
125 mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
126 tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
127 tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
128 tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
129 tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
130 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
131 then
132 printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
133 continue
134 fi
135 # assuming that at least TSTARTI and TSTOPI are consistent
136 #echo $rawfile
137 #echo $tstarti
138 #echo $tstopi
139 #echo $tstartf
140 #echo $tstopf
141 if [ $tstarti -gt 30000 ]
142 then
143 tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
144 tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec
145 #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
146 tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
147 else
148 tstart=`echo " $tstarti + $tstartf " | bc -l`
149 tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec
150 #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
151 tstop=`echo " $tstopi + $tstopf " | bc -l`
152 fi
153
154 # build query to update runinfo in DB
155 query="UPDATE RunInfo SET "
156
157 # get information
158 if [ -e $threshcurrentsfile ] && [ $threshnumerrors -eq 0 ]
159 then
160 thresholds1=( `root -q -b -l fact/processing/threshold.C\("\"$thresholdfile2\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` )
161 #root -q -b -l fact/threshold.C\("\"$thresholdfile2\""\,$tstart\,$tstop\) | grep "result"
162 #echo ${thresholds1[@]}
163 if [ "${thresholds1[0]}" == "" ]
164 then
165 query=$query"fThresholdMedMean=NULL"
166 else
167 query=$query"fThresholdMedMean="${thresholds1[0]}
168 fi
169 if [ "${thresholds1[1]}" == "" ]
170 then
171 query=$query", fThresholdMedRms=NULL"
172 else
173 query=$query", fThresholdMedRms="${thresholds1[1]}
174 fi
175 if [ "${thresholds1[2]}" == "" ]
176 then
177 query=$query", fThresholdMax=NULL"
178 else
179 query=$query", fThresholdMax="${thresholds1[2]}
180 fi
181 if [ "${thresholds1[3]}" == "" ]
182 then
183 query=$query", fThresholdAvgMean=NULL"
184 else
185 query=$query", fThresholdAvgMean="${thresholds1[3]}
186 fi
187 else
188 query=$query" fThresholdMedMean=NULL"
189 query=$query", fThresholdMedRms=NULL"
190 query=$query", fThresholdMax=NULL"
191 query=$query", fThresholdAvgMean=NULL"
192 fi
193
194 # get information
195 if [ -e $threshcurrentsfile2 ] && [ $threshnumerrors2 -eq 0 ]
196 then
197 thresholds2=( `root -q -b -l fact/processing/lastth.C\("\"$thresholdfile\""\,$tstart\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` )
198 #root -q -b -l fact/lastth.C\("\"$thresholdfile\""\,$tstart\) | grep "result"
199 #echo ${thresholds2[@]}
200 if [ "${thresholds2[0]}" == "" ]
201 then
202 query=$query", fThresholdMinSet=NULL"
203 else
204 query=$query", fThresholdMinSet="${thresholds2[0]}
205 fi
206 if [ "${thresholds2[1]}" == "" ]
207 then
208 query=$query", fThresholdMinTimeDiff=NULL"
209 else
210 query=$query", fThresholdMinTimeDiff="${thresholds2[1]}
211 fi
212 else
213 query=$query", fThresholdMinSet=NULL"
214 query=$query", fThresholdMinTimeDiff=NULL"
215 fi
216
217 # add where condition
218 query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
219
220 #echo $query
221 # send query to DB
222 sendquery >/dev/null
223 done
224done
225
226finish
227
228
Note: See TracBrowser for help on using the repository browser.