source: trunk/DataCheck/Processing/CheckRawData.sh@ 18692

Last change on this file since 18692 was 18661, checked in by Daniela Dorner, 10 years ago
fixed issue with times stamps that occured since update of gate and mysql version
  • Property svn:executable set to *
File size: 15.8 KB
Line 
1#!/bin/bash
2
3# completely new version of the script
4# running on newdaq to fill the information as fast as possible
5# ftools is not used in this script anymore
6
7# todo
8# tempfile for fitsdump -h output
9
10# option whether to fill all row or only those where information is missing
11# $doupdate might be given as environment variable
12if [ "$doupdate" = "" ]
13then
14 doupdate="yes" # update all entries (needed when new fields have been added)
15 doupdate="no" # fill only entries which are not yet existing (default)
16fi
17
18source `dirname $0`/../Sourcefile.sh
19printprocesslog "INFO starting $0 with options doupdate="$doupdate #" and skipmd5sum="$skipmd5sum
20
21# check if software is available
22if ! ls $factpath/fitsdump >/dev/null 2>&1
23then
24 printprocesslog "ERROR "$factpath"/fitsdump is not available."
25 finish
26fi
27
28# check if paths are available
29if ! ls $rawdata >/dev/null 2>&1
30then
31 printprocesslog "ERROR "$rawdata" is not available."
32 finish
33fi
34
35# get dates
36if [ "$certaindate" != "" ]
37then
38 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
39 if [ "$checkstring" = "" ]
40 then
41 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
42 finish
43 fi
44 getdates $certaindate
45else
46 # get all night
47 #getdates "all"
48 # get last 6 nights if hour between 7 and 19h, else only current night
49 getdates 6 7 19
50fi
51
52# do check for rawfiles of these dates
53for date in ${dates[@]}
54do
55 # get runnumber from date
56 runnumber=`echo $date | sed -e 's/\///g'`
57
58 night=
59 query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$runnumber
60 night=`sendquery`
61 if [ "$night" == "" ] && ls $rawdata/$date >/dev/null 2>&1
62 then
63 printprocesslog "INFO insert "$runnumber" to AuxDataInsertStatus"
64 query="INSERT AuxDataInsertStatus SET fNight="$runnumber", fPriority="$runnumber
65 sendquery >/dev/null 2>&1
66 fi
67 rawdir=$rawdata/$date
68 # check if data are available from that night
69 if ! [ -d $rawdir ]
70 then
71 printprocesslog "INFO "$rawdir" does not exist."
72 continue
73 else
74 printprocesslog "INFO processing "$rawdir"..."
75 fi
76
77 # find all fits.gz files starting with the oldest file
78 printprocesslog "INFO finding files to be checked in $rawdir..."
79 #fitsfiles=`ls $rawdir/*.fits 2>/dev/null | sort `
80 fitsfiles=( `find $rawdir -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
81
82 # loop to check files
83 for file in ${fitsfiles[@]}
84 do
85 printprocesslog "INFO checking file "$file
86 #echo "INFO checking file "$file
87 filenotclosed="no"
88
89 # check if it is drs file
90 isdrs=`ls $file | grep drs`
91 if [ "$isdrs" != "" ]
92 then
93 printprocesslog "INFO "$file" is a drs file. -> continue"
94 continue
95 fi
96
97 # get run and file number form filename
98 numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
99 runnumberfromname=`echo $numberfromname | cut -d_ -f1`
100 filenumberfromname=`echo $numberfromname | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'`
101 if [ "$runnumber" != "$runnumberfromname" ]
102 then
103 printprocesslog "ERROR for file "$file": runnumber from date ("$runnumber") and filename ("$runnumberfromname") don't agree."
104 fi
105
106 # check if entry already exists
107 query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
108 printprocesslog "DEBUG check if entry already exists in DB. QUERY: "$query
109 #result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
110 result3=`sendquery`
111
112 # only proceed with file
113 # if information is not yet in database
114 # and no update is wished ($doupdate)
115 if [ "$result3" != "" ] && [ "$doupdate" == "no" ]
116 then
117 printprocesslog "INFO "$file" has been inserted already. -> continue "
118 continue
119 fi
120
121 runtype=
122 # check if file was closed properly
123 if [ "`echo $file | grep -o drs`" == "drs" ]
124 then
125 nondrsfile=`echo $file | sed -e 's/[.]drs//g'`".fz" # fixme: maybe more flexible
126 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
127 checkstatus=$?
128 else
129 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
130 checkstatus=$?
131 fi
132 #if [ $checkstatus -gt 0 ]
133 #then
134 # echo "WARN: getting tstop for file "$file" failed: tstop: -"$tstop"-"`date`
135 #fi
136 if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
137 then
138 fileaccessed=`find $file -amin -30`
139 if ! [ "$fileaccessed" == "" ]
140 then
141 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
142 continue
143 else
144 printprocesslog "WARN "$file" has empty TSTOP but was not touched for 30 minutes"
145 filenotclosed="yes"
146 fi
147 fi
148
149 numdrsfiles=
150 step=
151 if [ "$filenotclosed" == "no" ]
152 then
153 # get run and file number from file
154 runnumberfromfile=`$factpath/fitsdump -h $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
155 filenumberfromfileorig=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
156 if [ "$runnumberfromfile" = "" ] || [ "$filenumberfromfileorig" = "" ]
157 then
158 printprocesslog "ERROR couldn't get run or file number from file name ("$file")."
159 fi
160 numberfromfile=$runnumberfromfile"_"`printf %03d $filenumberfromfileorig`
161 # compare numbers
162 if [ "$numberfromfile" != "$numberfromname" ]
163 then
164 printprocesslog "ERROR for file "$file": number from filename ("$numberfromname") and file ("$numberfromfile") don't agree."
165 fi
166
167 # get checksums from header
168 checksum=`$factpath/fitsdump -h $file 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
169 if [ "$checksum" == "" ]
170 then
171 printprocesslog "WARN checksum for file "$file" is empty."
172 fi
173 datasum=`$factpath/fitsdump -h $file 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
174 if [ "$datasum" == "" ]
175 then
176 printprocesslog "WARN datasum for file "$file" is empty."
177 fi
178
179 # check if this run has drs file
180 # in case file is available, get STEP from header
181 # in the very beginning only drs-files were existing
182 # in the beginning the keywords DRSCALIB and STEP were not existing
183 drsfile=`echo $file | sed -e 's/fits.fz/drs.fits/'`
184 numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
185 drscalib=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"`
186 if [ "$drscalib" == "T" ]
187 then
188 step=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"`
189 stepfromdrs=`$factpath/fitsdump -h $drsfile 2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"`
190 if [ "$stepfromdrs" != "$step" ]
191 then
192 printprocesslog "ERROR for file "$file" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree."
193 if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ]
194 then
195 step=$stepfromdrsfile
196 printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$file"."
197 fi
198 fi
199 if ! [ $numdrsfiles -eq 1 ]
200 then
201 printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
202 fi
203 if [ "$step" = "" ]
204 then
205 printprocesslog "ERROR file "$file" has drsfiles ("$numdrsfiles"), but step ("$step") is empty."
206 fi
207 else
208 if ! [ "$drscalib" == "F" ]
209 then
210 printprocesslog "WARN for file "$file" DRSCALIB is neither T nor F."
211 fi
212 fi
213
214 # get other variables from header
215 runtype=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
216 roi=`$factpath/fitsdump -h $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
217 roitm=`$factpath/fitsdump -h $file 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
218 numevents=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
219 numphys=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
220 numext1=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
221 numext2=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
222 numelp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
223 numilp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
224 numoth=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
225 numped=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
226 numtime=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
227 compiled=`$factpath/fitsdump -h $file 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
228 if ! [ "$compiled" == "" ]
229 then
230 compiletime=`date +'%F %H:%M:%S' --date="${compiled}" `
231 else
232 compiletime=
233 fi
234 revnum=`$factpath/fitsdump -h $file 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
235 # in newest data start time is in DATE-OBS
236 # in older data start time is in TSTART
237 # in the beginning TSTART was empty
238 runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
239 runstart2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
240 if [ "$runstart" == "" ]
241 then
242 if [ "$runstart2" == "" ]
243 then
244 # for very old data, but gives problems for newer data
245 #runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
246 #runstart="0000-00-00 00:00:00"
247 runstart=
248 else
249 runstart=$runstart2
250 fi
251 fi
252 # in newest data start time is in DATE-END
253 # in older data start time is in TSTOP
254 # in the beginning TSTOP was empty
255 runstop=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
256 runstop2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
257 if [ "$runstop" == "" ]
258 then
259 if [ "$runstop2" == "" ]
260 then
261 # for very old data, but gives problems for newer data
262 #runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
263 #runstop="0000-00-00 00:00:00"
264 runstop=
265 else
266 runstop=$runstop2
267 fi
268 fi
269 fi
270
271 # set runtype to 'unknown', if no runtype could be retrieved from file
272 if [ "$runtype" == "" ]
273 then
274 runtype="n/a"
275 fi
276 # on 15.11.2011 the runtypes had different names
277 if [ "$date" == "2011/11/15" ]
278 then
279 if [ "$runtype" == "drs-calib" ]
280 then
281 runtype="drs-gain"
282 fi
283 if [ "$runtype" == "drs-time-calib" ]
284 then
285 runtype="drs-time"
286 fi
287 if [ "$runtype" == "pedestal" ]
288 then
289 runtype="drs-pedestal"
290 fi
291 if [ "$runtype" == "light-pulser" ]
292 then
293 runtype="light-pulser-ext"
294 fi
295 if [ "$runtype" == "pedestal-on" ]
296 then
297 runtype="pedestal"
298 fi
299 fi
300 # get runtype
301 query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
302 printprocesslog "DEBUG get run type from DB. QUERY:"$query
303 result2=( `sendquery` )
304 if [ ${#result2} -eq 0 ]
305 then
306 printprocesslog "ERROR "$numberfromname": Could not query fRunTypeKey for runtype "$runtype" ."
307 continue
308 fi
309
310 # insert or update depending on whether run exists
311 if [ "$result3" == "" ]
312 then
313 query="INSERT"
314 querymid=" fNight="$runnumber", fRunID="$filenumberfromname", "
315 querystop=
316 else
317 query="UPDATE"
318 querymid=
319 querystop=" WHERE fNight="$runnumber" AND fRunID="$filenumberfromname
320 fi
321 query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
322 if [ "$filenotclosed" == "no" ]
323 then
324 if [ "$runstart" == "" ]
325 then
326 query=$query", fRunStart=NULL"
327 else
328 query=$query", fRunStart='"$runstart"'"
329 fi
330 if [ "$runstop" == "" ]
331 then
332 query=$query", fRunStop=NULL"
333 else
334 query=$query", fRunStop='"$runstop"'"
335 fi
336 if [ "$numevents" != "" ]
337 then
338 query=$query", fNumEvents="$numevents
339 fi
340 if [ "$roi" != "" ]
341 then
342 query=$query", fROI="$roi
343 fi
344 if [ "$roitm" != "" ]
345 then
346 query=$query", fROITimeMarker="$roitm
347 fi
348 if [ "$numphys" != "" ]
349 then
350 query=$query", fNumPhysicsTrigger="$numphys
351 fi
352 if [ "$numext1" != "" ]
353 then
354 query=$query", fNumExt1Trigger="$numext1
355 fi
356 if [ "$numext2" != "" ]
357 then
358 query=$query", fNumExt2Trigger="$numext2
359 fi
360 if [ "$numelp" != "" ]
361 then
362 query=$query", fNumELPTrigger="$numelp
363 fi
364 if [ "$numilp" != "" ]
365 then
366 query=$query", fNumILPTrigger="$numilp
367 fi
368 if [ "$numped" != "" ]
369 then
370 query=$query", fNumPedestalTrigger="$numped
371 fi
372 if [ "$numtime" != "" ]
373 then
374 query=$query", fNumTimeTrigger="$numtime
375 fi
376 if [ "$numoth" != "" ]
377 then
378 query=$query", fNumOtherTrigger="$numoth
379 fi
380 fi
381 if [ "$checksum" != "" ]
382 then
383 query=$query", fCheckSum='"$checksum"'"
384 fi
385 if [ "$datasum" != "" ]
386 then
387 query=$query", fDataSum='"$datasum"'"
388 fi
389 if [ "$numdrsfiles" != "" ]
390 then
391 query=$query", fHasDrsFile="$numdrsfiles
392 fi
393 if [ "$step" != "" ]
394 then
395 query=$query", fDrsStep="$step
396 fi
397 if [ "$compiletime" != "" ]
398 then
399 query=$query", fCompileTime='"$compiletime"'"
400 fi
401 if [ "$revnum" != "" ]
402 then
403 query=$query", fRevisionNumber='"$revnum"'"
404 fi
405 #query=$query", fFitsFileErrors="$numfitserrors
406 query=$query" "$querystop
407 # send query to DB
408 #echo $query
409 sendquery >/dev/null
410 done
411done
412
413finish
414
Note: See TracBrowser for help on using the repository browser.