source: trunk/DataCheck/QuickLook/Step1.sh@ 18082

Last change on this file since 18082 was 18068, checked in by Daniela Dorner, 10 years ago
follow links in find and other small fixes
  • Property svn:executable set to *
File size: 16.9 KB
Line 
1#!/bin/bash
2#
3
4# remarks:
5# rsync-server still used
6# move fileerror check to main-loop?
7
8source `dirname $0`/../Sourcefile.sh
9printprocesslog "INFO starting $0"
10
11# get date (before 18h there is no new data to be processed)
12if [ "$certaindate" != "" ]
13then
14 checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
15 if [ "$checkstring" = "" ]
16 then
17 echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
18 finish
19 fi
20 datepath=$certaindate
21else
22 datepath=`date --date="-19HOUR" +%Y/%m/%d`
23fi
24date=`echo $datepath | sed -e 's/\///g'`
25printprocesslog "INFO processing "$datepath
26
27auxpathnewdaq=/newdaq/aux/$datepath
28# create aux directory on daq, if not yet there
29auxpath=/loc_data/aux/$datepath
30makedir $auxpath >/dev/null
31# create path for info files needed for analysis
32infopath=$anapath/info/$datepath
33makedir $infopath >/dev/null
34echo "" > $infopath/runrow.txt
35# create path for callisto output
36calpath=$anapath/callisto/$datepath
37makedir $calpath >/dev/null
38rawpathnewdaq=/newdaq/raw/$datepath
39rawpath=/loc_data/raw/$datepath
40
41# needed auxiliary files:
42# drive file with information about current source position
43drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
44drivefilenewdaq=$auxpathnewdaq/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
45# drive file with information about tracking position
46drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
47drivefilenewdaq2=$auxpathnewdaq/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
48# file with magic weather information
49mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
50mweatherfilenewdaq=$auxpathnewdaq/${date}.MAGIC_WEATHER_DATA.fits
51# file with trigger rates
52ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
53ratesfilenewdaq=$auxpathnewdaq/${date}.FTM_CONTROL_TRIGGER_RATES.fits
54# file with trigger rates
55tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
56tempfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_TEMPERATURE.fits
57# file with trigger rates
58humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
59humfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_HUMIDITY.fits
60
61function rsync_aux_file()
62{
63 if ls $1 >/dev/null 2>&1
64 then
65 printprocesslog "INFO rsync "$1
66 # rsync
67 # from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/
68 # to daq (/daq = /loc_data on daq)
69 rsyncservernewdaq=`echo $1 | sed -e 's/^\//172.16.100.100::/'`
70 # old
71 #if ! rsync -a -T $rsynctempdir $1 $2
72 # new (workaround for problems on daq)
73 if ! rsync -a -T $rsynctempdir $rsyncservernewdaq $2
74 then
75 printprocesslog "WARN rsync of "$1" failed."
76 fi
77 else
78 printprocesslog "WARN "$1" missing."
79 fi
80}
81
82function check_daq()
83{
84 diskusage=( `df -P /raid10 | grep raid10 ` )
85 # check if more than 700 GB are left on /loc_data
86 if [ ${diskusage[3]} -lt $disklimitdaq ]
87 then
88 echo "WARN less than 700 left on /raid10 on node "$HOSTNAME
89 printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME
90 df -h /raid10
91 finish
92 fi
93}
94
95check_daq
96
97printprocesslog "INFO get lists of raw files on newdaq and daq"
98files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
99# to treat links use:
100#files=( `find -L $rawpathnewdaq -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
101
102if [ ${#files[@]} -eq 0 ]
103then
104 printprocesslog "INFO no raw files available yet for "$datepath
105 finish
106fi
107fileslocal=( `find -L $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null| sort` )
108callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null| sort` )
109# get number of dataruns from DB
110query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
111numdataruns=`sendquery`
112query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6"
113numlpruns=`sendquery`
114query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300"
115numpedruns=`sendquery`
116query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5"
117numdrstime=`sendquery`
118numpedruns=0
119#numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l`
120numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l`
121
122# create raw directory on daq, if not yet there
123makedir $rawpath >/dev/null
124
125#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
126printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
127
128while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
129do
130 # only continue with script
131 # when there is more than 10% space on daq
132 source `dirname $0`/../Sourcefile.sh
133 check_daq
134
135 numcalibrated=0
136 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
137 printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
138
139 rsync_aux_file $drivefilenewdaq $drivefile
140
141 # files on newdaq
142 for file in ${files[@]}
143 do
144 printprocesslog "processing "$file
145 #echo "processing "$file
146 localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
147
148 source `dirname $0`/../Sourcefile.sh
149 # check if file is already transferred
150 if ! ls $localfile >/dev/null 2>&1
151 then
152 # check if it is drs-file
153 # get stop time from raw-file
154 if [ "`echo $file | grep -o drs`" == "drs" ]
155 then
156 nondrs=`basename $file | sed -e 's/[.]drs//g'`
157 nondrsfile=`find -L $rawpath -name $nondrs.*z`
158 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
159 else
160 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
161 fi
162 # when stop time is 0, file is not closed
163 # when an error is returned the tstop is empty
164 if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
165 then
166 printprocesslog "WARN "$file" not yet closed."
167 # if a file is not closed and not touched for 30 minutes,
168 # it is assumed corrupted and still transferred
169 fileaccessed=`find $file -amin -30`
170 if ! [ "$fileaccessed" == "" ]
171 then
172 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
173 continue
174 else
175 printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
176 fileerror="yes"
177 fi
178 fi
179
180 # rsync
181 # from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/
182 # to daq (/daq = /loc_data on daq)
183 # to access rsync server via the dedicated network between
184 # daq and newdaq, use 172.16.100.100::newdaq
185 filersyncserver=`echo $file | sed -e 's/^\//172.16.100.100::/'`
186 # old
187 ##if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
188 #if ! rsync -a -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
189 # new
190 if ! rsync -a -W -T $rsynctempdir --bwlimit=$bwlimit $filersyncserver $localfile
191 then
192 printprocesslog "ERROR something went wrong with rsync of "$file
193 rm $localfile
194 continue
195 fi
196 printprocesslog "INFO "$file" rsynced successfully."
197 fi
198
199 # for .drs.fits files no further treatment needed
200 if [ "`echo $localfile | grep -o drs`" == "drs" ]
201 then
202 continue
203 fi
204
205# # temporary check
206# if [ "`echo $file | grep -o drs`" == "drs" ]
207# then
208# nondrs=`basename $file | sed -e 's/[.]drs//g'`
209# nondrsfile=`find -L $rawpath -name $nondrs.*z`
210# tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
211# else
212# tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
213# fi
214# # when stop time is 0, file is not closed
215# # when an error is returned the tstop is empty
216# if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
217# then
218# printprocesslog "WARN "$file" not yet closed. -> continue (temporary check)"
219# continue
220# fi
221# # end temporary check
222
223 # treat other files (.fits.fz)
224 runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
225 if [ "$runtype" != "data" ]
226 then
227 # skip a non-data run when it has not 1000 evts
228 # as this means probably an fad-loss
229 # and these runs are repeated in that case
230 numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
231 if [ $numevts -ne 1000 ]
232 then
233 printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")"
234 continue
235 fi
236 fi
237
238 # get run number
239 runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
240
241 # what is needed to process the different runs?
242 # P: run#(P), run#(drs-file)
243 # C: run#(C), run#(drs-file), run#(drs-time)
244 # D: run#(D), run#(drs-file), run#(drs-time), ?
245 # what is drs-file? pedestal, roi300, has drs.fits
246 callistolog=$calpath"/"$date"_"$runnum"-calibration.log"
247 case $runtype in
248 data) # treat D-runs
249 if [ "$fileerror" = "yes" ]
250 then
251 printprocesslog "INFO do not further process corrupted file "$localfile
252 fileerror=
253 continue
254 fi
255
256 # some accounting
257 printprocesslog "DEBUG counting callisto logs and set data files +1."
258 # get number of callisto logs
259 runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
260 # count runs to be calibrated
261 numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
262 printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]}
263
264 # do not overload system in case of a lot of files to be processed
265 # numruncallistos is set in setup.fact.lp.data
266 if [ $runcallistocount -ge $numruncallistos ]
267 then
268 printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue"
269 continue
270 fi
271
272 # starting calibration
273 if ! [ -e $callistolog ]
274 then
275 rsync_aux_file $drivefilenewdaq2 $drivefile2
276 rsync_aux_file $mweatherfilenewdaq $mweatherfile
277 rsync_aux_file $ratesfilenewdaq $ratesfile
278 rsync_aux_file $tempfilenewdaq $tempfile
279 rsync_aux_file $humfilenewdaq $humfile
280 if [ -e $drstime ]
281 then
282 calfile=$calpath"/"$date"_"$runnum"_C.root"
283 printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" outpath "$outpath" calfile "$calfile
284 #echo "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" outpath "$outpath" calfile "$calfile
285 `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $drstime $calpath $calfile &
286 fi
287 fi
288 continue
289 ;;
290 pedestal) # treat P-runs
291 roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
292 if [ $roi -eq 300 ]
293 then
294 # check drs-file
295 drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
296 if [ -e $drsfile ]
297 then
298 # set name of drs-file
299 drscalib=$drsfile
300 continue
301 #else
302 # not needed for QLA
303 #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
304 #if ! [ -e $callistolog ]
305 #then
306 # pedfile=$calpath"/"$date"_"$runnum"-pedestal.root"
307 # # count runs to be calibrated
308 # printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
309 # echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
310 # `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile &
311 #fi
312 fi
313 fi
314 ;;
315 light-pulser-ext) # treat C-runs
316 # do lp-treatment -> not needed for QLA
317 #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root"
318 #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
319 #if ! [ -e $callistolog ]
320 #then
321 # if [ -e $drstime ]
322 # then
323 # # count runs to be calibrated
324 # printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
325 # echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
326 # `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile &
327 # fi
328 #fi
329 ;;
330 drs-time) # treat C-runs
331 # do drs-timing calibration
332 drstime=$calpath"/"$date"_"$runnum"-drstime.root"
333 # starting calibration
334 numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
335 if ! [ -e $callistolog ]
336 then
337 # count runs to be calibrated
338 printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
339 #echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
340 `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime &
341 fi
342 ;;
343 drs-pedestal) # get drs-pedestal
344 roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
345 drs=`$factpath/fitsdump -h $localfile 2>/dev/null | grep DRSCALIB | grep -E -o " T " `
346 if [ $roi -eq 1024 ] && [ "$drs" == " T " ]
347 then
348 drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
349 fi
350 ;;
351 *) # other runs
352 printprocesslog "INFO file "$file" has runtype "$runtype" -> continue "
353 continue
354 ;;
355 esac
356 done
357 printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
358
359 # get new file lists
360 printprocesslog "INFO get new file lists for "$datepath
361 files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
362 fileslocal=( `find -L $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
363 callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` )
364 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
365 printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
366
367 # wait and get new file lists
368 update=
369 if [ ${#fileslocal[@]} -eq ${#files[@]} ]
370 then
371 printprocesslog "INFO wait 60 seconds."
372 sleep 60
373 printprocesslog "INFO get new file lists for "$datepath
374 files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
375 fileslocal=( `find -L $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
376 callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` )
377 fi
378 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
379 printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
380 sleep 30
381 printprocesslog "INFO sleep 30"
382done
383
384
385
Note: See TracBrowser for help on using the repository browser.