1 | #!/bin/bash
|
---|
2 |
|
---|
3 | today=`date +%F`
|
---|
4 | logfile=/home/`whoami`/DataCheck/log/FillAuxData$today.log
|
---|
5 |
|
---|
6 | # steps:
|
---|
7 | # select run# from DB - no, date is given
|
---|
8 | # get daily files
|
---|
9 | # select file# from DB
|
---|
10 | # select start and stop from DB
|
---|
11 | # get info from files
|
---|
12 | # fill into into DB
|
---|
13 |
|
---|
14 |
|
---|
15 | doupdate="yes" # update all entries
|
---|
16 | doupdate="no" # fill only entries which are not yet existing
|
---|
17 |
|
---|
18 | echo "" >> $logfile 2>&1
|
---|
19 | echo "" >> $logfile 2>&1
|
---|
20 | echo "" >> $logfile 2>&1
|
---|
21 | echo `date`"executing "$0"..." >> $logfile 2>&1
|
---|
22 | echo "=====> doupdate: "$doupdate >> $logfile 2>&1
|
---|
23 |
|
---|
24 | password=`cat /home/fact/DataCheck/.pw`
|
---|
25 | if [ "$password" == "" ]
|
---|
26 | then
|
---|
27 | echo "please insert password in .pw file"
|
---|
28 | fi
|
---|
29 |
|
---|
30 | # setup to use ftools
|
---|
31 | export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
|
---|
32 | source $HEADAS/headas-init.sh
|
---|
33 |
|
---|
34 | # get last 2 nights
|
---|
35 | dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
|
---|
36 | #dates=( "2011/11/29" )
|
---|
37 | # problems with 2011/11/29
|
---|
38 |
|
---|
39 | # do rsync for rawfiles of these dates
|
---|
40 | for date in ${dates[@]}
|
---|
41 | do
|
---|
42 | echo "" >> $logfile 2>&1
|
---|
43 | echo "" >> $logfile 2>&1
|
---|
44 | echo "" >> $logfile 2>&1
|
---|
45 | auxdir=/loc_data/aux/$date
|
---|
46 | runnumber=`echo $date | sed -e 's/\///g'`
|
---|
47 | echo `date`": processing files in "$auxdir >> $logfile 2>&1
|
---|
48 | # check if data are available from that night
|
---|
49 | if ! [ -d $auxdir ]
|
---|
50 | then
|
---|
51 | echo `date`": no data available in "$auxdir >> $logfile 2>&1
|
---|
52 | continue
|
---|
53 | fi
|
---|
54 |
|
---|
55 | # get daily fits files
|
---|
56 | trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
|
---|
57 | if ! [ -e $trackingfile ]
|
---|
58 | then
|
---|
59 | echo $trackingfile" not found"
|
---|
60 | fi
|
---|
61 | triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
|
---|
62 | if ! [ -e $triggerratefile ]
|
---|
63 | then
|
---|
64 | echo $triggerratefile" not found"
|
---|
65 | fi
|
---|
66 | thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
|
---|
67 | if ! [ -e $thresholdfile ]
|
---|
68 | then
|
---|
69 | echo $thresholdfile" not found"
|
---|
70 | fi
|
---|
71 | biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
|
---|
72 | if ! [ -e $biasvoltagefile ]
|
---|
73 | then
|
---|
74 | echo $biasvoltagefile" not found"
|
---|
75 | fi
|
---|
76 | biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
|
---|
77 | if ! [ -e $biascurrentsfile ]
|
---|
78 | then
|
---|
79 | echo $biascurrentsfile" not found"
|
---|
80 | fi
|
---|
81 |
|
---|
82 | # get file numbers from DB
|
---|
83 | query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber
|
---|
84 | filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>> $logfile` )
|
---|
85 | for filenum in ${filenumbers[@]}
|
---|
86 | do
|
---|
87 | #echo $filenum
|
---|
88 | #query2="SELECT fRunStart from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
|
---|
89 | #starttime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query2" 2>> $logfile`
|
---|
90 | #query3="SELECT fRunStop from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
|
---|
91 | #stoptime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query3" 2>> $logfile`
|
---|
92 | #echo "start: "$starttime
|
---|
93 | #echo "stop: "$stoptime
|
---|
94 | rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits
|
---|
95 | if ! [ -e $trackingfile ]
|
---|
96 | then
|
---|
97 | echo "ERROR: "$rawfile" not found."
|
---|
98 | continue
|
---|
99 | #else
|
---|
100 | # echo $rawfile
|
---|
101 | fi
|
---|
102 | checkfitsfile=`fverify $rawfile 2>> $logfile | grep '0 error(s)'`
|
---|
103 | if [ "$checkfitsfile" == "" ]
|
---|
104 | then
|
---|
105 | numfitserrors=1
|
---|
106 | echo "WARN: "$rawfile" probably corrupted."
|
---|
107 | continue
|
---|
108 | fi
|
---|
109 | runtype=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
|
---|
110 | #echo " runtype: "$runtype
|
---|
111 | #/home/fact/FACT++/fitsdump -h -t Events $rawfile
|
---|
112 | mjdref=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
|
---|
113 | tstarti=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
|
---|
114 | tstartf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
|
---|
115 | tstopi=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
|
---|
116 | tstopf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
|
---|
117 | if [ "$tstarti" == "" ] || [ "$tstopi" == "" ]
|
---|
118 | then
|
---|
119 | echo "WARN: "$rawfile" has start or stop time = 0."
|
---|
120 | continue
|
---|
121 | fi
|
---|
122 | # calculate start and stop times
|
---|
123 | tstart=`echo " $tstarti + $mjdref + $tstartf " | bc -l`
|
---|
124 | tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.00011574 " | bc -l` # 10 sec
|
---|
125 | #tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.000023148 " | bc -l` # 2 sec
|
---|
126 | tstop=`echo " $tstopi + $mjdref + $tstopf " | bc -l`
|
---|
127 | #echo " tstart in mjd: "$tstart
|
---|
128 | #echo " tstop in mjd: "$tstop
|
---|
129 | query0="UPDATE RunInfo SET "
|
---|
130 | queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
|
---|
131 |
|
---|
132 | # get information from tracking
|
---|
133 | #echo $trackingfile
|
---|
134 | ramin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
|
---|
135 | ramax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
|
---|
136 | ramean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
|
---|
137 | decmin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
|
---|
138 | decmax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
|
---|
139 | decmean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
|
---|
140 | #echo " ra: "$ramin" - "$ramean" - "$ramax
|
---|
141 | if [ "$ramin" == "$ramax" ] && [ "$ramean" != "" ]
|
---|
142 | then
|
---|
143 | query0=$query0" fRightAscension="$ramean
|
---|
144 | else
|
---|
145 | query0=$query0" fRightAscension=NULL"
|
---|
146 | fi
|
---|
147 | #echo " dec: "$decmin" - "$decmean" - "$decmax
|
---|
148 | if [ "$decmin" == "$decmax" ] && [ "$decmean" != "" ]
|
---|
149 | then
|
---|
150 | query0=$query0", fDeclination="$decmean
|
---|
151 | else
|
---|
152 | query0=$query0", fDeclination=NULL"
|
---|
153 | fi
|
---|
154 |
|
---|
155 | # get information from trigger
|
---|
156 | #echo $triggerratefile
|
---|
157 | #ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat -
|
---|
158 | ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
159 | ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
160 | ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
161 | ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
162 | #echo " rate: "$ratemin" - "$ratemedian" - "$ratemean" - "$ratemax
|
---|
163 | if [ "$ratemedian" != "0" ]
|
---|
164 | then
|
---|
165 | query0=$query0", fTriggerRateMedian="$ratemedian
|
---|
166 | else
|
---|
167 | query0=$query0", fTriggerRateMedian=NULL"
|
---|
168 | fi
|
---|
169 |
|
---|
170 | # get information from trigger
|
---|
171 | #echo $thresholdfile
|
---|
172 | #echo "ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
|
---|
173 | threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
174 | threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
175 | threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
176 | threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
177 | #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
|
---|
178 | if [ "$threshmean" == "0" ]
|
---|
179 | then
|
---|
180 | #echo "ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
|
---|
181 | threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
182 | threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
183 | threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
184 | threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
185 | #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
|
---|
186 | # is this taking into account the 160 values in a row?
|
---|
187 | fi
|
---|
188 | query0=$query0", fThresholdMedian="$threshmedian
|
---|
189 |
|
---|
190 | # # get information from bias: Uref
|
---|
191 | # #echo $biasvoltagefile
|
---|
192 | # #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
|
---|
193 | # biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
194 | # biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
195 | # biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
196 | # biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
197 | # echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
|
---|
198 | # if [ "$biasvoltmean" == "0" ]
|
---|
199 | # then
|
---|
200 | # #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
|
---|
201 | # biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
202 | # biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
203 | # biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
204 | # biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
205 | # echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
|
---|
206 | # # is this taking into account the 160 values in a row?
|
---|
207 | # fi
|
---|
208 |
|
---|
209 | # get information from bias: U
|
---|
210 | #echo $biasvoltagefile
|
---|
211 | #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
|
---|
212 | biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
213 | biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
214 | biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
215 | biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
216 | #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
|
---|
217 | if [ "$biasvoltmean" == "0" ]
|
---|
218 | then
|
---|
219 | #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
|
---|
220 | biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
221 | biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
222 | biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
223 | biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
|
---|
224 | #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
|
---|
225 | # is this taking into account the 160 values in a row?
|
---|
226 | fi
|
---|
227 | query0=$query0", fBiasVoltageMedian="$biasvoltmedian
|
---|
228 |
|
---|
229 | query0=$query0" "$queryend
|
---|
230 |
|
---|
231 | if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >> $logfile 2>&1
|
---|
232 | then
|
---|
233 | echo "insert/update of "$runnumber" "$filenum" to mysql failed" >> $logfile 2>&1
|
---|
234 | echo "Q0: "$query0" --- query failed."
|
---|
235 | fi
|
---|
236 | done
|
---|
237 | done
|
---|
238 |
|
---|
239 |
|
---|