source: trunk/DataCheck/Processing/CheckRawData.sh@ 13057

Last change on this file since 13057 was 13057, checked in by Daniela Dorner, 13 years ago
added priority to status queries
  • Property svn:executable set to *
File size: 16.6 KB
Line 
1#!/bin/bash
2
3# this script has been written to run on La Palma on the machine data
4# i.e. paths are only working on this machine
5# the script starts from the zipped files
6# this causes a delay until files are in the database
7# because they have to be rsynced and zipped first (see RsyncRawData.sh, ZipRawData.sh)
8
9# missing
10# entry in status table
11
12# options:
13skipmd5sum="no" # fill md5 sums in any case
14skipmd5sum="iffilled" # fill md5 sum only if they are not yet in db # default
15#skipmd5sum="yes" # do not fill md5 sums in any case
16
17doupdate="yes" # update all entries
18doupdate="no" # fill only entries which are not yet existing #default
19
20source `dirname $0`/../Sourcefile.sh
21printprocesslog "INFO starting $0 with options doupdate="$doupdate" and skipmd5sum="$skipmd5sum
22
23# setup to use ftools
24source $HEADAS/headas-init.sh
25
26# check if software is available
27if ! ls $factpath/fitsdump >/dev/null 2>&1
28then
29 printprocesslog "ERROR "$factpath"/fitsdump is not available."
30 finish
31fi
32
33# check if paths are available
34if ! ls /daq/raw >/dev/null 2>&1
35then
36 printprocesslog "ERROR /daq/raw is not available."
37 finish
38fi
39if ! ls /loc_data/raw >/dev/null 2>&1
40then
41 printprocesslog "ERROR /loc_data/raw is not available."
42 finish
43fi
44if ! ls /loc_data/zipraw >/dev/null 2>&1
45then
46 printprocesslog "ERROR /loc_data/zipraw is not available."
47 finish
48fi
49
50# get last 3 nights
51dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
52#dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw
53
54# do check for rawfiles of these dates
55for date in ${dates[@]}
56do
57 ziprawdir=/loc_data/zipraw/$date
58 # check if data are available from that night
59 if ! [ -d $ziprawdir ]
60 then
61 printprocesslog "INFO "$ziprawdir" does not exist."
62 continue
63 else
64 printprocesslog "INFO processing "$ziprawdir"..."
65 fi
66
67 # find all fits.gz files starting with the oldest file
68 printprocesslog "INFO finding files to be checked in $ziprawdir..."
69 fitsgzfiles=`find $ziprawdir -type f -name '*.fits.gz'| sort `
70
71 # get runnumber from date
72 runnumber=`echo $date | sed -e 's/\///g'`
73
74 # loop to check files
75 for file in $fitsgzfiles
76 do
77 printprocesslog "INFO checking file "$file
78 #echo "INFO checking file "$file
79
80 # raw and original file
81 rawfile=`echo $file | sed -e 's/zipraw/raw/' -e 's/fits[.]gz/fits/'`
82 origfile=`echo $rawfile | sed -e 's/loc_data/daq/'`
83
84 # check if it is drs file
85 isdrs=`ls $file | grep drs`
86 if [ "$isdrs" != "" ]
87 then
88 printprocesslog "INFO "$file" is a drs file. -> continue"
89 continue
90 fi
91
92 # check if file is already finished
93 # original file on daq (if data was taken on daq
94 if [ -e $origfile ]
95 then
96 # check if raw file was changed in the last 30 minutes
97 isnew=`find $origfile -cmin -30`
98 if [ "$isnew" != "" ]
99 then
100 printprocesslog "WARN "$origfile" is not older than 30 min. -> continue"
101 continue
102 fi
103
104 # get time of last modification as seconds since Epoch for both files
105 timeorig=`stat -c %Y $origfile`
106 timecopy=`stat -c %Y $rawfile`
107 # compare times
108 if ! [ $timeorig -eq $timecopy ]
109 then
110 # if times are not the same, the file is still open => no check
111 printprocesslog "INFO file "$rawfile" not yet closed. -> continue"
112 continue
113 fi
114 else
115 # if the origfile doesn't exist, the data was probably written not on daq but on data
116 printprocesslog "INFO file "$rawfile" was probably taken on data and not daq."
117 fi
118
119 # get run and file number form filename
120 runnumbererror="no"
121 numbererror="no"
122 numberfromname=`echo $file | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'`
123 runnumberfromname=`echo $numberfromname | cut -d_ -f1`
124 filenumberfromname=`echo $numberfromname | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'`
125 if [ "$runnumber" != "$runnumberfromname" ]
126 then
127 runnumbererror="yes"
128 printprocesslog "ERROR for file "$file": runnumber from date ("$runnumber") and filename ("$runnumberfromname") don't agree."
129 fi
130
131 # check if entry already exists
132 query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
133 printprocesslog "DEBUG check if entry already exists in DB. QUERY: "$query
134 #result3=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query3"`
135 result3=`sendquery`
136
137 # only proceed with file
138 # if information is not yet in database
139 # and no update is wished ($doupdate)
140 if [ "$result3" != "" ] && [ "$doupdate" == "no" ]
141 then
142 printprocesslog "INFO "$file" has been inserted already. -> continue "
143 continue
144 fi
145
146 runtype=
147 # check if fits file is corrupted
148 numfitserrors=0
149 #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'`
150 numfitserrors=`fverify $rawfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'`
151 #if [ "$checkfitsfile" == "" ]
152 if [ $numfitserrors -gt 0 ]
153 then
154 printprocesslog "WARN "$rawfile" has "$numfitserrors" fitserror(s). "
155 #numfitserrors=1
156 fi
157
158 if [ $numfitserrors -eq 0 ]
159 then
160 # get run and file number from file
161 runnumberfromfile=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NIGHT | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]'`
162 filenumberfromfileorig=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep RUNID | grep -E -o '[0-9]{1,3}'`
163 if [ "$runnumberfromfile" = "" ] || [ "$filenumberfromfileorig" = "" ]
164 then
165 printprocesslog "ERROR couldn't get run or file number from file name ("$file")."
166 fi
167 numberfromfile=$runnumberfromfile"_"`printf %03d $filenumberfromfileorig`
168 # compare numbers
169 if [ "$numberfromfile" != "$numberfromname" ]
170 then
171 numbererror="yes"
172 printprocesslog "ERROR for file "$file": number from filename ("$numberfromname") and file ("$numberfromfile") don't agree."
173 fi
174
175 # check if this run has drs file
176 # in case file is available, get STEP from header
177 # in the very beginning only drs-files were existing
178 # in the beginning the keywords DRSCALIB and STEP were not existing
179 numdrsfiles=
180 step=
181 drsfile=`echo $file | sed -e 's/fits/drs.fits/'`
182 numdrsfiles=`ls $drsfile 2>/dev/null | wc -l`
183 drscalib=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DRSCALIB | grep -E -o "['][TF][']" | sed -e "s/'//g"`
184 if [ "$drscalib" == "T" ]
185 then
186 step=`$factpath/fitsdump -h -t Events $drsfile 2>/dev/null | grep STEP | grep -E -o "['][012][']" | sed -e "s/'//g"`
187 if ! [ $numdrsfiles -eq 1 ]
188 then
189 printprocesslog "ERROR for file "$file" number of drsfiles ("$numdrsfiles") and information from header ("$drscalib") don't agree."
190 fi
191 else
192 if ! [ "$drscalib" == "F" ]
193 then
194 printprocesslog "WARN for file "$file" DRSCALIB is neither T nor F."
195 fi
196 fi
197
198 # get other variables from header
199 runtype=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
200 #echo "runtype for file "$file": "$runtype
201 roi=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'`
202 roitm=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'`
203 numevents=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
204 numphys=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'`
205 numext1=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGEXT1' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
206 numext2=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGEXT2' | grep -E -o '[ ][0-9]+[ ]' | sed -e 's/\ //g'`
207 numelp=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'`
208 numilp=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'`
209 numoth=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'`
210 numped=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'`
211 numtime=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'`
212 compiled=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'COMPILED' | grep -E -o "['][a-zA-Z]+[ ][ 12][0-9][ ]20[0-9][0-9][ ][0-2][0-9]:[0-5][0-9]:[0-5][0-9][']" | sed -e "s/'//g"`
213 if ! [ "$compiled" == "" ]
214 then
215 compiletime=`date +'%F %H:%M:%S' --date="${compiled}" `
216 else
217 compiletime=
218 fi
219 revnum=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep 'REVISION' | grep -E -o "['][0-9]+[:]?[0-9]*[MSP]*[']" | sed -e "s/'//g"`
220 # in newest data start time is in DATE-OBS
221 # in older data start time is in TSTART
222 # in the beginning TSTART was empty
223 runstart=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
224 runstart2=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
225 if [ "$runstart" == "" ]
226 then
227 if [ "$runstart2" == "" ]
228 then
229 runstart=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
230 else
231 runstart=$runstart2
232 fi
233 fi
234 # in newest data start time is in DATE-END
235 # in older data start time is in TSTOP
236 # in the beginning TSTOP was empty
237 runstop=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
238 runstop2=`$factpath/fitsdump -h -t Events $file 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'`
239 if [ "$runstop" == "" ]
240 then
241 if [ "$runstop2" == "" ]
242 then
243 runstop=`stat $file 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'`
244 else
245 runstop=$runstop2
246 fi
247 fi
248 fi
249
250 # set runtype to 'unknown', if no runtype could be retrieved from file
251 if [ "$runtype" == "" ]
252 then
253 runtype="n/a"
254 fi
255 # on 15.11.2011 the runtypes had different names
256 if [ "$date" == "2011/11/15" ]
257 then
258 if [ "$runtype" == "drs-calib" ]
259 then
260 runtype="drs-gain"
261 fi
262 if [ "$runtype" == "drs-time-calib" ]
263 then
264 runtype="drs-time"
265 fi
266 if [ "$runtype" == "pedestal" ]
267 then
268 runtype="drs-pedestal"
269 fi
270 if [ "$runtype" == "light-pulser" ]
271 then
272 runtype="light-pulser-ext"
273 fi
274 if [ "$runtype" == "pedestal-on" ]
275 then
276 runtype="pedestal"
277 fi
278 fi
279 # get runtype
280 query="SELECT fRunTypeKEY FROM RunType WHERE fRunTypeName='"$runtype"'"
281 printprocesslog "DEBUG get run type from DB. QUERY:"$query
282 #result2=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query2"` )
283 result2=( `sendquery` )
284 if [ ${#result2} -eq 0 ]
285 then
286 printprocesslog "ERROR "$numberfromname": Could not query fRunTypeKey for runtype "$runtype" ."
287 continue
288 fi
289
290 # check if entry has already checksums
291 query="SELECT fNight FROM RunInfo WHERE Concat(fNight, '_', LPAD(fRunID, 3, 0))='"$numberfromname"'"
292 query=$query" AND NOT ISNULL(fMd5sumRaw) AND NOT ISNULL(fMd5sumRawZip)"
293 printprocesslog "DEBUG check if md5sums are alreay in DB. QUERY:"$query
294 #result5=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query5"`
295 result5=`sendquery`
296
297 # get md5sums of raw and zip file
298 # to safe time for tests and update this can be skipped ($skipmd5sum)
299 md5sum=
300 md5sumzip=
301 if [ "$skipmd5sum" == "no" ] || [ "$skipmd5sum" == "iffilled" ]
302 then
303 #echo "skip: "$skipmd5sum >> $logfile 2>&1
304 #echo "res5: -"$result5"-" >> $logfile 2>&1
305 #if [ "$skipmd5sum" == "no" ] || [ "$result5$skipmd5sum" == "iffilled" ]
306 if [ "$skipmd5sum" == "no" ] || [ "$result5" == "" ]
307 then
308 #zipfile=`echo $file | sed -e 's/raw/zipraw/' -e 's/fits/fits.gz/'`
309 # only do the md5sum if the zipfile is already available
310 if ls $file >/dev/null 2>&1
311 then
312 printprocesslog "INFO calculation md5sum for rawfile "$rawfile
313 md5sum=`md5sum $rawfile | cut -d' ' -f1`
314 printprocesslog "INFO calculation md5sum for zipfile "$file
315 md5sumzip=`md5sum $file | cut -d' ' -f1`
316 fi
317 fi
318 fi
319
320 # insert or update depending on whether run exists
321 if [ "$result3" == "" ]
322 then
323 query="INSERT RawFileAvailISDCStatus SET fNight="$runnumber", fPriority="$runnumber", fRunID="$filenumberfromname
324 sendquery >/dev/null
325 query="INSERT RawFileRsyncedISDCStatus SET fNight="$runnumber", fPriority="$runnumber", fRunID="$filenumberfromname
326 sendquery >/dev/null
327 query="INSERT RawFileAvailWueStatus SET fNight="$runnumber", fPriority="$runnumber", fRunID="$filenumberfromname
328 sendquery >/dev/null
329 query="INSERT"
330 querymid=" fNight="$runnumber", fRunID="$filenumberfromname", "
331 querystop=
332 else
333 query="UPDATE"
334 querymid=
335 querystop=" WHERE fNight="$runnumber" AND fRunID="$filenumberfromname
336 fi
337 query=$query" RunInfo SET "$querymid" fRunTypeKey="${result2[0]}
338 if [ $numfitserrors -eq 0 ]
339 then
340 query=$query", fRunStart='"$runstart"', fRunStop='"$runstop"'"
341 if [ "$numevents" != "" ]
342 then
343 query=$query", fNumEvents="$numevents
344 fi
345 if [ "$roi" != "" ]
346 then
347 query=$query", fROI="$roi
348 fi
349 if [ "$roitm" != "" ]
350 then
351 query=$query", fROITimeMarker="$roitm
352 fi
353 if [ "$numphys" != "" ]
354 then
355 query=$query", fNumPhysicsTrigger="$numphys
356 fi
357 if [ "$numext1" != "" ]
358 then
359 query=$query", fNumExt1Trigger="$numext1
360 fi
361 if [ "$numext2" != "" ]
362 then
363 query=$query", fNumExt2Trigger="$numext2
364 fi
365 if [ "$numelp" != "" ]
366 then
367 query=$query", fNumELPTrigger="$numelp
368 fi
369 if [ "$numilp" != "" ]
370 then
371 query=$query", fNumILPTrigger="$numilp
372 fi
373 if [ "$numped" != "" ]
374 then
375 query=$query", fNumPedestalTrigger="$numped
376 fi
377 if [ "$numtime" != "" ]
378 then
379 query=$query", fNumTimeTrigger="$numtime
380 fi
381 if [ "$numoth" != "" ]
382 then
383 query=$query", fNumOtherTrigger="$numoth
384 fi
385 fi
386 if [ "$md5sum" != "" ]
387 then
388 query=$query", fMd5sumRaw='"$md5sum"', fMd5sumRawZip='"$md5sumzip"'"
389 fi
390 if [ "$numdrsfiles" != "" ]
391 then
392 query=$query", fHasDrsFile="$numdrsfiles
393 fi
394 if [ "$step" != "" ]
395 then
396 query=$query", fDrsStep="$step
397 fi
398 if [ "$compiletime" != "" ]
399 then
400 query=$query", fCompileTime='"$compiletime"'"
401 fi
402 if [ "$revnum" != "" ]
403 then
404 query=$query", fRevisionNumber='"$revnum"'"
405 fi
406 query=$query", fFitsFileErrors="$numfitserrors
407 query=$query" "$querystop
408 # send query to DB
409 sendquery >/dev/null
410 done
411done
412
413finish
414
Note: See TracBrowser for help on using the repository browser.