source: branches/trigger_burst_research/Processing/CheckRawData.sh@ 18803

Last change on this file since 18803 was 18093, checked in by Daniela Dorner, 10 years ago
fixed bug: drs runs had been excluded by last fix
  • Property svn:executable set to *
File size: 15.5 KB
Line 
1#!/bin/bash
2
3# completely new version of the script
4# running on newdaq to fill the information as fast as possible
5# ftools is not used in this script anymore
6
7# todo
8# tempfile for fitsdump -h output
9
10# option whether to fill all row or only those where information is missing
11# $doupdate might be given as environment variable
12if [ "$doupdate" = "" ]
13then
14 doupdate="yes" # update all entries (needed when new fields have been added)
15 doupdate="no" # fill only entries which are not yet existing (default)
16fi
17
18source `dirname $0`/../Sourcefile.sh
19printprocesslog "INFO starting $0 with options doupdate="$doupdate #" and skipmd5sum="$skipmd5sum
20
21# check if software is available
22if ! ls $factpath/fitsdump >/dev/null 2>&1
23then
24 printprocesslog "ERROR "$factpath"/fitsdump is not available."
25 finish
26fi
27
28# check if paths are available
29if ! ls $rawdata >/dev/null 2>&1
30then
31 printprocesslog "ERROR "$rawdata" is not available."
32 finish
33fi
34
35# get dates
36if [ "$certaindate" != "" ]
37then
38 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
39 if [ "$checkstring" = "" ]
40 then
41 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
42 finish
43 fi
44 getdates $certaindate
45else
46 # get all night
47 #getdates "all"
48 # get last 6 nights if hour between 7 and 19h, else only current night
49 getdates 6 7 19
50fi
51
52# do check for rawfiles of these dates
53for date in ${dates[@]}
54do
55 # get runnumber from date
56 runnumber=`echo $date | sed -e 's/\///g'`
57
58 night=
59 query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$runnumber
60 night=`sendquery`
61 if [ "$night" == "" ] && ls $rawdata/$date >/dev/null 2>&1
62 then
63 printprocesslog "INFO insert "$runnumber" to AuxDataInsertStatus"
64 query="INSERT AuxDataInsertStatus SET fNight="$runnumber", fPriority="$runnumber
65 sendquery >/dev/null 2>&1
66 fi
67 rawdir=$rawdata/$date
68 # check if data are available from that night
69 if ! [ -d $rawdir ]
70 then
71 printprocesslog "INFO "$rawdir" does not exist."
72 continue
73 else
74 printprocesslog "INFO processing "$rawdir"..."
75 fi
76
77 # find all fits.gz files starting with the oldest file
78 printprocesslog "INFO finding files to be checked in $rawdir..."
79 #fitsfiles=`ls $rawdir/*.fits 2>/dev/null | sort `
80 fitsfiles=( `find $rawdir -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
81
82 # loop to check files
83 for file in ${fitsfiles[@]}
84 do
85 printprocesslog "INFO checking file "$file
86 #echo "INFO checking file "$file
87 filenotclosed="no"
88
89 # check if it is drs file
90 isdrs=`ls $file | grep drs`
91 if [ "$isdrs" != "" ]
92 then
93 printprocesslog "INFO "$file" is a drs file. -> continue"
94 continue
95 fi
96
97 # get run and file number form filename
98 numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
99 runnumberfromname=`echo $numberfromname | cut -d_ -f1`
100 filenumberfromname=`echo $numberfromname | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'`
101 if [ "$runnumber" != "$runnumberfromname" ]
102 then
103 printprocesslog "ERROR for file "$file": runnumber from date ("$runnumber") and filename ("$runnumberfromname") don't agree."
104 fi
105
106 # check if entry already exists
107 query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
108 printprocesslog "DEBUG check if entry already exists in DB. QUERY: "$query
109 #result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
110 result3=`sendquery`
111
112 # only proceed with file
113 # if information is not yet in database
114 # and no update is wished ($doupdate)
115 if [ "$result3" != "" ] && [ "$doupdate" == "no" ]
116 then
117 printprocesslog "INFO "$file" has been inserted already. -> continue "
118 continue
119 fi
120
121 runtype=
122 # check if file was closed properly
123 if [ "`echo $file | grep -o drs`" == "drs" ]
124 then
125 nondrsfile=`echo $file | sed -e 's/[.]drs//g'`".fz" # fixme: maybe more flexible
126 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
127 checkstatus=$?
128 else
129 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
130 checkstatus=$?
131 fi
132 #if [ $checkstatus -gt 0 ]
133 #then
134 # echo "WARN: getting tstop for file "$file" failed: tstop: -"$tstop"-"`date`
135 #fi
136 if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
137 then
138 fileaccessed=`find $file -amin -30`
139 if ! [ "$fileaccessed" == "" ]
140 then
141 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
142 continue
143 else
144 printprocesslog "WARN "$file" has empty TSTOP but was not touched for 30 minutes"
145 filenotclosed="yes"
146 fi
147 fi
148
149 numdrsfiles=
150 step=
151 if [ "$filenotclosed" == "no" ]
152 then
153 # get run and file number from file
154 runnumberfromfile=`$factpath/fitsdump -h $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
155 filenumberfromfileorig=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
156 if [ "$runnumberfromfile" = "" ] || [ "$filenumberfromfileorig" = "" ]
157 then
158 printprocesslog "ERROR couldn't get run or file number from file name ("$file")."
159 fi
160 numberfromfile=$runnumberfromfile"_"`printf %03d $filenumberfromfileorig`
161 # compare numbers
162 if [ "$numberfromfile" != "$numberfromname" ]
163 then
164 printprocesslog "ERROR for file "$file": number from filename ("$numberfromname") and file ("$numberfromfile") don't agree."
165 fi
166
167 # get checksums from header
168 checksum=`$factpath/fitsdump -h $file 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
169 if [ "$checksum" == "" ]
170 then
171 printprocesslog "WARN checksum for file "$file" is empty."
172 fi
173 datasum=`$factpath/fitsdump -h $file 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
174 if [ "$datasum" == "" ]
175 then
176 printprocesslog "WARN datasum for file "$file" is empty."
177 fi
178
179 # check if this run has drs file
180 # in case file is available, get STEP from header
181 # in the very beginning only drs-files were existing
182 # in the beginning the keywords DRSCALIB and STEP were not existing
183 drsfile=`echo $file | sed -e 's/fits.fz/drs.fits/'`
184 numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
185 drscalib=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"`
186 if [ "$drscalib" == "T" ]
187 then
188 step=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"`
189 stepfromdrs=`$factpath/fitsdump -h $drsfile 2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"`
190 if [ "$stepfromdrs" != "$step" ]
191 then
192 printprocesslog "ERROR for file "$file" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree."
193 if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ]
194 then
195 step=$stepfromdrsfile
196 printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$file"."
197 fi
198 fi
199 if ! [ $numdrsfiles -eq 1 ]
200 then
201 printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
202 fi
203 if [ "$step" = "" ]
204 then
205 printprocesslog "ERROR file "$file" has drsfiles ("$numdrsfiles"), but step ("$step") is empty."
206 fi
207 else
208 if ! [ "$drscalib" == "F" ]
209 then
210 printprocesslog "WARN for file "$file" DRSCALIB is neither T nor F."
211 fi
212 fi
213
214 # get other variables from header
215 runtype=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
216 roi=`$factpath/fitsdump -h $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
217 roitm=`$factpath/fitsdump -h $file 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
218 numevents=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
219 numphys=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
220 numext1=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
221 numext2=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
222 numelp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
223 numilp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
224 numoth=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
225 numped=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
226 numtime=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
227 compiled=`$factpath/fitsdump -h $file 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
228 if ! [ "$compiled" == "" ]
229 then
230 compiletime=`date +'%F %H:%M:%S' --date="${compiled}" `
231 else
232 compiletime=
233 fi
234 revnum=`$factpath/fitsdump -h $file 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
235 # in newest data start time is in DATE-OBS
236 # in older data start time is in TSTART
237 # in the beginning TSTART was empty
238 runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
239 runstart2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
240 if [ "$runstart" == "" ]
241 then
242 if [ "$runstart2" == "" ]
243 then
244 # for very old data, but gives problems for newer data
245 #runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
246 runstart="0000-00-00 00:00:00"
247 else
248 runstart=$runstart2
249 fi
250 fi
251 # in newest data start time is in DATE-END
252 # in older data start time is in TSTOP
253 # in the beginning TSTOP was empty
254 runstop=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
255 runstop2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
256 if [ "$runstop" == "" ]
257 then
258 if [ "$runstop2" == "" ]
259 then
260 # for very old data, but gives problems for newer data
261 #runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
262 runstop="0000-00-00 00:00:00"
263 else
264 runstop=$runstop2
265 fi
266 fi
267 fi
268
269 # set runtype to 'unknown', if no runtype could be retrieved from file
270 if [ "$runtype" == "" ]
271 then
272 runtype="n/a"
273 fi
274 # on 15.11.2011 the runtypes had different names
275 if [ "$date" == "2011/11/15" ]
276 then
277 if [ "$runtype" == "drs-calib" ]
278 then
279 runtype="drs-gain"
280 fi
281 if [ "$runtype" == "drs-time-calib" ]
282 then
283 runtype="drs-time"
284 fi
285 if [ "$runtype" == "pedestal" ]
286 then
287 runtype="drs-pedestal"
288 fi
289 if [ "$runtype" == "light-pulser" ]
290 then
291 runtype="light-pulser-ext"
292 fi
293 if [ "$runtype" == "pedestal-on" ]
294 then
295 runtype="pedestal"
296 fi
297 fi
298 # get runtype
299 query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
300 printprocesslog "DEBUG get run type from DB. QUERY:"$query
301 result2=( `sendquery` )
302 if [ ${#result2} -eq 0 ]
303 then
304 printprocesslog "ERROR "$numberfromname": Could not query fRunTypeKey for runtype "$runtype" ."
305 continue
306 fi
307
308 # insert or update depending on whether run exists
309 if [ "$result3" == "" ]
310 then
311 query="INSERT"
312 querymid=" fNight="$runnumber", fRunID="$filenumberfromname", "
313 querystop=
314 else
315 query="UPDATE"
316 querymid=
317 querystop=" WHERE fNight="$runnumber" AND fRunID="$filenumberfromname
318 fi
319 query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
320 if [ "$filenotclosed" == "no" ]
321 then
322 query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'"
323 if [ "$numevents" != "" ]
324 then
325 query=$query", fNumEvents="$numevents
326 fi
327 if [ "$roi" != "" ]
328 then
329 query=$query", fROI="$roi
330 fi
331 if [ "$roitm" != "" ]
332 then
333 query=$query", fROITimeMarker="$roitm
334 fi
335 if [ "$numphys" != "" ]
336 then
337 query=$query", fNumPhysicsTrigger="$numphys
338 fi
339 if [ "$numext1" != "" ]
340 then
341 query=$query", fNumExt1Trigger="$numext1
342 fi
343 if [ "$numext2" != "" ]
344 then
345 query=$query", fNumExt2Trigger="$numext2
346 fi
347 if [ "$numelp" != "" ]
348 then
349 query=$query", fNumELPTrigger="$numelp
350 fi
351 if [ "$numilp" != "" ]
352 then
353 query=$query", fNumILPTrigger="$numilp
354 fi
355 if [ "$numped" != "" ]
356 then
357 query=$query", fNumPedestalTrigger="$numped
358 fi
359 if [ "$numtime" != "" ]
360 then
361 query=$query", fNumTimeTrigger="$numtime
362 fi
363 if [ "$numoth" != "" ]
364 then
365 query=$query", fNumOtherTrigger="$numoth
366 fi
367 fi
368 if [ "$checksum" != "" ]
369 then
370 query=$query", fCheckSum='"$checksum"'"
371 fi
372 if [ "$datasum" != "" ]
373 then
374 query=$query", fDataSum='"$datasum"'"
375 fi
376 if [ "$numdrsfiles" != "" ]
377 then
378 query=$query", fHasDrsFile="$numdrsfiles
379 fi
380 if [ "$step" != "" ]
381 then
382 query=$query", fDrsStep="$step
383 fi
384 if [ "$compiletime" != "" ]
385 then
386 query=$query", fCompileTime='"$compiletime"'"
387 fi
388 if [ "$revnum" != "" ]
389 then
390 query=$query", fRevisionNumber='"$revnum"'"
391 fi
392 #query=$query", fFitsFileErrors="$numfitserrors
393 query=$query" "$querystop
394 # send query to DB
395 #echo $query
396 sendquery >/dev/null
397 done
398done
399
400finish
401
Note: See TracBrowser for help on using the repository browser.