source: trunk/DataCheck/Processing/CheckRawData.sh@ 19952

Last change on this file since 19952 was 19202, checked in by Daniela Dorner, 6 years ago
added filling of FileID
  • Property svn:executable set to *
File size: 16.0 KB
Line 
1#!/bin/bash
2
3# completely new version of the script
4# running on newdaq to fill the information as fast as possible
5# ftools is not used in this script anymore
6
7# todo
8# tempfile for fitsdump -h output
9
10# option whether to fill all row or only those where information is missing
11# $doupdate might be given as environment variable
12if [ "$doupdate" = "" ]
13then
14 doupdate="yes" # update all entries (needed when new fields have been added)
15 doupdate="no" # fill only entries which are not yet existing (default)
16fi
17
18source `dirname $0`/../Sourcefile.sh
19printprocesslog "INFO starting $0 with options doupdate="$doupdate #" and skipmd5sum="$skipmd5sum
20
21# check if software is available
22if ! ls $factpath/fitsdump >/dev/null 2>&1
23then
24 printprocesslog "ERROR "$factpath"/fitsdump is not available."
25 finish
26fi
27
28# check if paths are available
29if ! ls $rawdata >/dev/null 2>&1
30then
31 printprocesslog "ERROR "$rawdata" is not available."
32 finish
33fi
34
35# get dates
36if [ "$certaindate" != "" ]
37then
38 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
39 if [ "$checkstring" = "" ]
40 then
41 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
42 finish
43 fi
44 getdates $certaindate
45else
46 # get all night
47 #getdates "all"
48 # get last 6 nights if hour between 7 and 19h, else only current night
49 getdates 6 7 19
50fi
51
52# do check for rawfiles of these dates
53for date in ${dates[@]}
54do
55 # get runnumber from date
56 runnumber=`echo $date | sed -e 's/\///g'`
57
58 night=
59 query="SELECT fNight FROM AuxDataInsertStatus WHERE fNight="$runnumber
60 night=`sendquery`
61 if [ "$night" == "" ] && ls $rawdata/$date >/dev/null 2>&1
62 then
63 printprocesslog "INFO insert "$runnumber" to AuxDataInsertStatus"
64 query="INSERT AuxDataInsertStatus SET fNight="$runnumber", fPriority="$runnumber
65 sendquery >/dev/null 2>&1
66 fi
67 rawdir=$rawdata/$date
68 # check if data are available from that night
69 if ! [ -d $rawdir ]
70 then
71 printprocesslog "INFO "$rawdir" does not exist."
72 continue
73 else
74 printprocesslog "INFO processing "$rawdir"..."
75 fi
76
77 # find all fits.gz files starting with the oldest file
78 printprocesslog "INFO finding files to be checked in $rawdir..."
79 #fitsfiles=`ls $rawdir/*.fits 2>/dev/null | sort `
80 fitsfiles=( `find $rawdir -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
81
82 # loop to check files
83 for file in ${fitsfiles[@]}
84 do
85 printprocesslog "INFO checking file "$file
86 #echo "INFO checking file "$file
87 filenotclosed="no"
88
89 # check if it is drs file
90 isdrs=`ls $file | grep drs`
91 if [ "$isdrs" != "" ]
92 then
93 printprocesslog "INFO "$file" is a drs file. -> continue"
94 continue
95 fi
96
97 # get run and file number form filename
98 numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
99 runnumberfromname=`echo $numberfromname | cut -d_ -f1`
100 filenumberfromname=`echo $numberfromname | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'`
101 if [ "$runnumber" != "$runnumberfromname" ]
102 then
103 printprocesslog "ERROR for file "$file": runnumber from date ("$runnumber") and filename ("$runnumberfromname") don't agree."
104 fi
105
106 # check if entry already exists
107 query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
108 printprocesslog "DEBUG check if entry already exists in DB. QUERY: "$query
109 #result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
110 result3=`sendquery`
111
112 # only proceed with file
113 # if information is not yet in database
114 # and no update is wished ($doupdate)
115 if [ "$result3" != "" ] && [ "$doupdate" == "no" ]
116 then
117 printprocesslog "INFO "$file" has been inserted already. -> continue "
118 continue
119 fi
120
121 runtype=
122 # check if file was closed properly
123 if [ "`echo $file | grep -o drs`" == "drs" ]
124 then
125 nondrsfile=`echo $file | sed -e 's/[.]drs//g'`".fz" # fixme: maybe more flexible
126 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
127 checkstatus=$?
128 else
129 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
130 checkstatus=$?
131 fi
132 #if [ $checkstatus -gt 0 ]
133 #then
134 # echo "WARN: getting tstop for file "$file" failed: tstop: -"$tstop"-"`date`
135 #fi
136 if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
137 then
138 fileaccessed=`find $file -amin -30`
139 if ! [ "$fileaccessed" == "" ]
140 then
141 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
142 continue
143 else
144 printprocesslog "WARN "$file" has empty TSTOP but was not touched for 30 minutes"
145 filenotclosed="yes"
146 fi
147 fi
148
149 numdrsfiles=
150 step=
151 if [ "$filenotclosed" == "no" ]
152 then
153 # get run and file number from file
154 runnumberfromfile=`$factpath/fitsdump -h $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
155 filenumberfromfileorig=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
156 if [ "$runnumberfromfile" = "" ] || [ "$filenumberfromfileorig" = "" ]
157 then
158 printprocesslog "ERROR couldn't get run or file number from file name ("$file")."
159 fi
160 numberfromfile=$runnumberfromfile"_"`printf %03d $filenumberfromfileorig`
161 # compare numbers
162 if [ "$numberfromfile" != "$numberfromname" ]
163 then
164 printprocesslog "ERROR for file "$file": number from filename ("$numberfromname") and file ("$numberfromfile") don't agree."
165 fi
166
167 # get checksums from header
168 checksum=`$factpath/fitsdump -h $file 2>/dev/null | grep CHECKSUM | grep -E -o '[a-zA-Z0-9]{16}'`
169 if [ "$checksum" == "" ]
170 then
171 printprocesslog "WARN checksum for file "$file" is empty."
172 fi
173 datasum=`$factpath/fitsdump -h $file 2>/dev/null | grep DATASUM | grep -E -o '[0-9]{1,10}'`
174 if [ "$datasum" == "" ]
175 then
176 printprocesslog "WARN datasum for file "$file" is empty."
177 fi
178
179 # check if this run has drs file
180 # in case file is available, get STEP from header
181 # in the very beginning only drs-files were existing
182 # in the beginning the keywords DRSCALIB and STEP were not existing
183 drsfile=`echo $file | sed -e 's/fits.fz/drs.fits/'`
184 numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
185 drscalib=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSCALIB | grep -E -o "[\ ][TF][\ ]" | sed -e "s/\ //g"`
186 if [ "$drscalib" == "T" ]
187 then
188 step=`$factpath/fitsdump -h $file 2>/dev/null | grep DRSSTEP | grep -E -o "[\ ][012][\ ]" | sed -e "s/\ //g"`
189 stepfromdrs=`$factpath/fitsdump -h $drsfile 2>/dev/null | grep STEP | grep -E -o "[\ ][012][\ ]?" | sed -e "s/\ //g"`
190 if [ "$stepfromdrs" != "$step" ]
191 then
192 printprocesslog "ERROR for file "$file" step from drsfile ("$stepfromdrs") and from file ("$step") do not agree."
193 if [ "$stepfromdrsfile" != "" ] && [ "$step" == "" ]
194 then
195 step=$stepfromdrsfile
196 printprocesslog "WARN setting drsstep from drsfile ("$stepfromdrs") although value differs from the one in file "$file"."
197 fi
198 fi
199 if ! [ $numdrsfiles -eq 1 ]
200 then
201 printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
202 fi
203 if [ "$step" = "" ]
204 then
205 printprocesslog "ERROR file "$file" has drsfiles ("$numdrsfiles"), but step ("$step") is empty."
206 fi
207 else
208 if ! [ "$drscalib" == "F" ]
209 then
210 printprocesslog "WARN for file "$file" DRSCALIB is neither T nor F."
211 fi
212 fi
213
214 # get other variables from header
215 runtype=`$factpath/fitsdump -h $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
216 roi=`$factpath/fitsdump -h $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
217 roitm=`$factpath/fitsdump -h $file 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
218 numevents=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
219 numphys=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
220 numext1=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
221 numext2=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
222 numelp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
223 numilp=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
224 numoth=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
225 numped=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
226 numtime=`$factpath/fitsdump -h $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
227 compiled=`$factpath/fitsdump -h $file 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
228 if ! [ "$compiled" == "" ]
229 then
230 compiletime=`date +'%F %H:%M:%S' --date="${compiled}" `
231 else
232 compiletime=
233 fi
234 revnum=`$factpath/fitsdump -h $file 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
235 # in newest data start time is in DATE-OBS
236 # in older data start time is in TSTART
237 # in the beginning TSTART was empty
238 runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
239 runstart2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
240 if [ "$runstart" == "" ]
241 then
242 if [ "$runstart2" == "" ]
243 then
244 # for very old data, but gives problems for newer data
245 #runstart=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
246 #runstart="0000-00-00 00:00:00"
247 runstart=
248 else
249 runstart=$runstart2
250 fi
251 fi
252 # in newest data start time is in DATE-END
253 # in older data start time is in TSTOP
254 # in the beginning TSTOP was empty
255 runstop=`$factpath/fitsdump -h $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
256 runstop2=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
257 if [ "$runstop" == "" ]
258 then
259 if [ "$runstop2" == "" ]
260 then
261 # for very old data, but gives problems for newer data
262 #runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
263 #runstop="0000-00-00 00:00:00"
264 runstop=
265 else
266 runstop=$runstop2
267 fi
268 fi
269 fi
270
271 # set runtype to 'unknown', if no runtype could be retrieved from file
272 if [ "$runtype" == "" ]
273 then
274 runtype="n/a"
275 fi
276 # on 15.11.2011 the runtypes had different names
277 if [ "$date" == "2011/11/15" ]
278 then
279 if [ "$runtype" == "drs-calib" ]
280 then
281 runtype="drs-gain"
282 fi
283 if [ "$runtype" == "drs-time-calib" ]
284 then
285 runtype="drs-time"
286 fi
287 if [ "$runtype" == "pedestal" ]
288 then
289 runtype="drs-pedestal"
290 fi
291 if [ "$runtype" == "light-pulser" ]
292 then
293 runtype="light-pulser-ext"
294 fi
295 if [ "$runtype" == "pedestal-on" ]
296 then
297 runtype="pedestal"
298 fi
299 fi
300 # get runtype
301 query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
302 printprocesslog "DEBUG get run type from DB. QUERY:"$query
303 result2=( `sendquery` )
304 if [ ${#result2} -eq 0 ]
305 then
306 printprocesslog "ERROR "$numberfromname": Could not query fRunTypeKey for runtype "$runtype" ."
307 continue
308 fi
309
310 # insert or update depending on whether run exists
311 if [ "$result3" == "" ]
312 then
313 query="INSERT"
314 querymid=" fNight="$runnumber", fRunID="$filenumberfromname", FileID="`echo $runnumber | cut -c 3-8``printf %03d $filenumberfromname`", "
315 querystop=
316 else
317 query="UPDATE"
318 querymid=
319 querystop=" WHERE fNight="$runnumber" AND fRunID="$filenumberfromname
320 fi
321 query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
322
323 # some information is only available at the end of the run
324 # the fits header is updated then
325 if [ "$filenotclosed" == "no" ]
326 then
327 if [ "$runstart" == "" ]
328 then
329 query=$query", fRunStart=NULL"
330 else
331 query=$query", fRunStart='"$runstart"'"
332 fi
333 if [ "$runstop" == "" ]
334 then
335 query=$query", fRunStop=NULL"
336 else
337 query=$query", fRunStop='"$runstop"'"
338 fi
339 if [ "$numevents" != "" ]
340 then
341 query=$query", fNumEvents="$numevents
342 fi
343 if [ "$numphys" != "" ]
344 then
345 query=$query", fNumPhysicsTrigger="$numphys
346 fi
347 if [ "$numext1" != "" ]
348 then
349 query=$query", fNumExt1Trigger="$numext1
350 fi
351 if [ "$numext2" != "" ]
352 then
353 query=$query", fNumExt2Trigger="$numext2
354 fi
355 if [ "$numelp" != "" ]
356 then
357 query=$query", fNumELPTrigger="$numelp
358 fi
359 if [ "$numilp" != "" ]
360 then
361 query=$query", fNumILPTrigger="$numilp
362 fi
363 if [ "$numped" != "" ]
364 then
365 query=$query", fNumPedestalTrigger="$numped
366 fi
367 if [ "$numtime" != "" ]
368 then
369 query=$query", fNumTimeTrigger="$numtime
370 fi
371 if [ "$numoth" != "" ]
372 then
373 query=$query", fNumOtherTrigger="$numoth
374 fi
375 fi
376 if [ "$roi" != "" ]
377 then
378 query=$query", fROI="$roi
379 fi
380 if [ "$roitm" != "" ]
381 then
382 query=$query", fROITimeMarker="$roitm
383 fi
384 if [ "$checksum" != "" ]
385 then
386 query=$query", fCheckSum='"$checksum"'"
387 fi
388 if [ "$datasum" != "" ]
389 then
390 query=$query", fDataSum='"$datasum"'"
391 fi
392 if [ "$numdrsfiles" != "" ]
393 then
394 query=$query", fHasDrsFile="$numdrsfiles
395 fi
396 if [ "$step" != "" ]
397 then
398 query=$query", fDrsStep="$step
399 fi
400 if [ "$compiletime" != "" ]
401 then
402 query=$query", fCompileTime='"$compiletime"'"
403 fi
404 if [ "$revnum" != "" ]
405 then
406 query=$query", fRevisionNumber='"$revnum"'"
407 fi
408 #query=$query", fFitsFileErrors="$numfitserrors
409 query=$query" "$querystop
410 # send query to DB
411 #echo $query
412 sendquery >/dev/null
413 done
414done
415
416finish
417
Note: See TracBrowser for help on using the repository browser.