source: trunk/DataCheck/FillAuxData.sh@ 12727

Last change on this file since 12727 was 12691, checked in by Daniela Dorner, 13 years ago
added filling of information into db
  • Property svn:executable set to *
File size: 14.5 KB
Line 
1#!/bin/bash
2
3today=`date +%F`
4logfile=/home/`whoami`/DataCheck/log/FillAuxData$today.log
5
6# steps:
7# select run# from DB - no, date is given
8# get daily files
9# select file# from DB
10# select start and stop from DB
11# get info from files
12# fill into into DB
13
14
15doupdate="yes" # update all entries
16doupdate="no" # fill only entries which are not yet existing
17
18echo "" >> $logfile 2>&1
19echo "" >> $logfile 2>&1
20echo "" >> $logfile 2>&1
21echo `date`"executing "$0"..." >> $logfile 2>&1
22echo "=====> doupdate: "$doupdate >> $logfile 2>&1
23
24password=`cat /home/fact/DataCheck/.pw`
25if [ "$password" == "" ]
26then
27 echo "please insert password in .pw file"
28fi
29
30# setup to use ftools
31export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
32source $HEADAS/headas-init.sh
33
34# get last 2 nights
35dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
36#dates=( "2011/11/29" )
37# problems with 2011/11/29
38
39# do rsync for rawfiles of these dates
40for date in ${dates[@]}
41do
42 echo "" >> $logfile 2>&1
43 echo "" >> $logfile 2>&1
44 echo "" >> $logfile 2>&1
45 auxdir=/loc_data/aux/$date
46 runnumber=`echo $date | sed -e 's/\///g'`
47 echo `date`": processing files in "$auxdir >> $logfile 2>&1
48 # check if data are available from that night
49 if ! [ -d $auxdir ]
50 then
51 echo `date`": no data available in "$auxdir >> $logfile 2>&1
52 continue
53 fi
54
55 # get daily fits files
56 trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
57 if ! [ -e $trackingfile ]
58 then
59 echo $trackingfile" not found"
60 fi
61 triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
62 if ! [ -e $triggerratefile ]
63 then
64 echo $triggerratefile" not found"
65 fi
66 thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
67 if ! [ -e $thresholdfile ]
68 then
69 echo $thresholdfile" not found"
70 fi
71 biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
72 if ! [ -e $biasvoltagefile ]
73 then
74 echo $biasvoltagefile" not found"
75 fi
76 biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
77 if ! [ -e $biascurrentsfile ]
78 then
79 echo $biascurrentsfile" not found"
80 fi
81
82 # get file numbers from DB
83 query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber
84 filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>> $logfile` )
85 for filenum in ${filenumbers[@]}
86 do
87 #echo $filenum
88 #query2="SELECT fRunStart from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
89 #starttime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query2" 2>> $logfile`
90 #query3="SELECT fRunStop from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
91 #stoptime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query3" 2>> $logfile`
92 #echo "start: "$starttime
93 #echo "stop: "$stoptime
94 rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits
95 if ! [ -e $trackingfile ]
96 then
97 echo "ERROR: "$rawfile" not found."
98 continue
99 #else
100 # echo $rawfile
101 fi
102 checkfitsfile=`fverify $rawfile 2>> $logfile | grep '0 error(s)'`
103 if [ "$checkfitsfile" == "" ]
104 then
105 numfitserrors=1
106 echo "WARN: "$rawfile" probably corrupted."
107 continue
108 fi
109 runtype=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
110 #echo " runtype: "$runtype
111 #/home/fact/FACT++/fitsdump -h -t Events $rawfile
112 mjdref=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
113 tstarti=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
114 tstartf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
115 tstopi=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
116 tstopf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
117 if [ "$tstarti" == "" ] || [ "$tstopi" == "" ]
118 then
119 echo "WARN: "$rawfile" has start or stop time = 0."
120 continue
121 fi
122 # calculate start and stop times
123 tstart=`echo " $tstarti + $mjdref + $tstartf " | bc -l`
124 tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.00011574 " | bc -l` # 10 sec
125 #tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.000023148 " | bc -l` # 2 sec
126 tstop=`echo " $tstopi + $mjdref + $tstopf " | bc -l`
127 #echo " tstart in mjd: "$tstart
128 #echo " tstop in mjd: "$tstop
129 query0="UPDATE RunInfo SET "
130 queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
131
132 # get information from tracking
133 #echo $trackingfile
134 ramin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
135 ramax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
136 ramean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
137 decmin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
138 decmax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
139 decmean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
140 #echo " ra: "$ramin" - "$ramean" - "$ramax
141 if [ "$ramin" == "$ramax" ] && [ "$ramean" != "" ]
142 then
143 query0=$query0" fRightAscension="$ramean
144 else
145 query0=$query0" fRightAscension=NULL"
146 fi
147 #echo " dec: "$decmin" - "$decmean" - "$decmax
148 if [ "$decmin" == "$decmax" ] && [ "$decmean" != "" ]
149 then
150 query0=$query0", fDeclination="$decmean
151 else
152 query0=$query0", fDeclination=NULL"
153 fi
154
155 # get information from trigger
156 #echo $triggerratefile
157 #ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat -
158 ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
159 ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
160 ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
161 ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
162 #echo " rate: "$ratemin" - "$ratemedian" - "$ratemean" - "$ratemax
163 if [ "$ratemedian" != "0" ]
164 then
165 query0=$query0", fTriggerRateMedian="$ratemedian
166 else
167 query0=$query0", fTriggerRateMedian=NULL"
168 fi
169
170 # get information from trigger
171 #echo $thresholdfile
172 #echo "ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
173 threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
174 threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
175 threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
176 threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
177 #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
178 if [ "$threshmean" == "0" ]
179 then
180 #echo "ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
181 threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
182 threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
183 threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
184 threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
185 #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
186 # is this taking into account the 160 values in a row?
187 fi
188 query0=$query0", fThresholdMedian="$threshmedian
189
190# # get information from bias: Uref
191# #echo $biasvoltagefile
192# #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
193# biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
194# biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
195# biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
196# biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
197# echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
198# if [ "$biasvoltmean" == "0" ]
199# then
200# #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
201# biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
202# biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
203# biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
204# biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
205# echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
206# # is this taking into account the 160 values in a row?
207# fi
208
209 # get information from bias: U
210 #echo $biasvoltagefile
211 #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
212 biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
213 biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
214 biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
215 biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
216 #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
217 if [ "$biasvoltmean" == "0" ]
218 then
219 #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
220 biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
221 biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
222 biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
223 biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
224 #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
225 # is this taking into account the 160 values in a row?
226 fi
227 query0=$query0", fBiasVoltageMedian="$biasvoltmedian
228
229 query0=$query0" "$queryend
230
231 if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >> $logfile 2>&1
232 then
233 echo "insert/update of "$runnumber" "$filenum" to mysql failed" >> $logfile 2>&1
234 echo "Q0: "$query0" --- query failed."
235 fi
236 done
237done
238
239
Note: See TracBrowser for help on using the repository browser.