source: trunk/DataCheck/QuickLook/Step1.sh@ 15560

Last change on this file since 15560 was 15558, checked in by Daniela Dorner, 12 years ago
bugfix in disk-usage-check
  • Property svn:executable set to *
File size: 14.7 KB
Line 
1#!/bin/bash
2#
3source `dirname $0`/../Sourcefile.sh
4printprocesslog "INFO starting $0"
5
6# get date (before 18h there is no new data to be processed)
7datepath=`date --date="-19HOUR" +%Y/%m/%d`
8date=`date --date="-19HOUR" +%Y%m%d`
9#datepath="2013/04/17"
10#date="20130417"
11printprocesslog "INFO processing "$datepath
12night=`echo $datepath | sed -e 's/\///g'`
13
14auxpathnewdaq=/newdaq/aux/$datepath
15# create aux directory on daq, if not yet there
16auxpath=/loc_data/aux/$datepath
17makedir $auxpath
18# create path for info files needed for analysis
19infopath=$anapath/info/$datepath
20makedir $infopath
21echo "" > $infopath/runrow.txt
22# create path for sequence files
23seqpath=$anapath/sequences/$datepath
24makedir $seqpath
25rawpathnewdaq=/newdaq/raw/$datepath
26rawpath=/loc_data/raw/$datepath
27
28# needed auxiliary files:
29# drive file with information about current source position
30drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
31drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
32# drive file with information about tracking position
33drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
34drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
35# file with magic weather information
36mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
37mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
38# file with trigger rates
39ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
40ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
41# file with trigger rates
42tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
43tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
44# file with trigger rates
45humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
46humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits
47
48function rsync_aux_file()
49{
50 if ls $1 >/dev/null 2>&1
51 then
52 printprocesslog "INFO rsync "$1
53 #if ! rsync -av --stats $1 $2
54 #if ! rsync -av $1 $2
55 if ! rsync -a $1 $2
56 then
57 printprocesslog "WARN rsync of "$1" failed."
58 fi
59 else
60 printprocesslog "WARN "$1" missing."
61 fi
62}
63
64function check_daq()
65{
66 diskusage=( `df -P /raid10 | grep raid10 ` )
67 # check if more than 700 GB are left on /loc_data
68 if [ ${diskusage[3]} -lt 800000000 ]
69 then
70 echo "WARN less than 700 left on /raid10 on node "$HOSTNAME
71 printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME
72 df -h /raid10
73 finish
74 fi
75}
76
77check_daq
78
79printprocesslog "INFO get lists of raw files on newdaq and daq"
80files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
81if [ ${#files[@]} -eq 0 ]
82then
83 printprocesslog "INFO no raw files available yet for "$datepath
84 finish
85fi
86fileslocal=( `find $rawpath -type f | sort` )
87callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
88numdataruns=0
89
90# create raw directory on daq, if not yet there
91makedir $rawpath
92
93#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
94printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
95
96while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ]
97do
98 # only continue with script
99 # when there is more than 10% space on daq
100 check_daq
101
102 numdataruns=0
103 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
104 printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
105
106 rsync_aux_file $drivefilenewdaq $drivefile
107
108 # files on newdaq
109 for file in ${files[@]}
110 do
111 printprocesslog "processing "$file
112 localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
113 #echo "processing "$file" "$localfile
114
115 if [ "`echo $file | grep -o drs`" == "drs" ]
116 then
117 nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
118 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
119 else
120 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
121 fi
122 if [ "$tstop" == "0" ]
123 then
124 printprocesslog "WARN "$file" not yet closed."
125 fileaccessed=`find $file -amin -30`
126 if ! [ "$fileaccessed" == "" ]
127 then
128 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
129 continue
130 else
131 printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
132 fileerror="yes"
133 fi
134 fi
135
136 source `dirname $0`/../Sourcefile.sh
137 if ! ls $localfile >/dev/null 2>&1
138 then
139 #if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
140 if ! rsync -a --bwlimit=$bwlimit $file $localfile
141 then
142 printprocesslog "ERROR something went wrong with rsync of "$file
143 rm $localfile
144 continue
145 fi
146 printprocesslog "INFO "$file" rsynced successfully."
147 fi
148
149
150 if [ "`echo $localfile | grep -o drs`" != "drs" ]
151 then
152 runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
153 runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
154 roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
155 numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
156 printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi" numevts "$numevts
157 if [ "$runtype" == "drs-time-upshifted" ]
158 then
159 printprocesslog "INFO file "$file" has runtype drs-time-upshifted -> continue "
160 continue
161 fi
162 if [ "$runtype" == "data" ]
163 then
164 if [ "$fileerror" = "yes" ]
165 then
166 printprocesslog "INFO do not further process corrupted file "$localfile
167 fileerror=
168 continue
169 fi
170 seqfile=$seqpath/${night}_${runnum}.seq
171 printprocesslog "INFO write data-seq "$seqfile
172 echo "# written by automatic analysis in LP" >$seqfile
173 echo "" >> $seqfile
174 echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
175 echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
176 echo "" >> $seqfile
177 echo "DrsSequence: "$drsseq >> $seqfile
178 echo "" >> $seqfile
179 echo "CalRuns: "`echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'` >> $seqfile
180 echo "PedRuns: "`echo $runrow | grep -E -o '[0-9]{3}pedestal300' | sed -e 's/pedestal300//g'` >> $seqfile
181 echo "DatRuns: "$runnum >> $seqfile
182 echo "" >> $seqfile
183 echo "DrsFiles: "$drsfile >> $seqfile
184 echo "" >> $seqfile
185 echo "#DrsFile: "$drsfile >> $seqfile
186 echo "" >> $seqfile
187
188# tstopi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
189# tstopf=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
190# tstop=${tstopi}${tstopf}
191# coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop} 2>/dev/null | tail -1 2>&1` )
192# if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
193# then
194# printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
195# #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
196# continue
197# fi
198# if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
199# then
200# printprocesslog "WARN coordinates "${coordinates[@]}
201# #echo "WARN coordinates "${coordinates[@]}
202# continue
203# fi
204# printprocesslog "DEBUG coordinates "${coordinates[@]}
205# query="SELECT fSourceKEY FROM scheduling.source WHERE "
206# query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
207# query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
208# sourcekey=`sendquery`
209# if [ "$sourcekey" == "" ]
210# then
211# printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
212# fi
213
214 printprocesslog "INFO counting callisto logs and data files +1."
215 # get number of callisto logs
216 callistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
217 # count data runs
218 numdataruns=`echo " $numdataruns + 1 " | bc -l`
219 #echo "numdata +1"
220
221 #echo "cal: "$callistocount" numdat: "$numdataruns" numcallog: "${#callistofiles[@]}
222 printprocesslog "INFO running callistos: "$callistocount" #data-runs: "$numdataruns" #callisto-logs: "${#callistofiles[@]}
223 # do not overload system in case of a lot of files to be processed
224 # callistocount is set in setup.fact.lp.data
225 if [ $callistocount -ge $numcallistos ]
226 then
227 printprocesslog "INFO "$callistocount" RunCallisto.sh are running -> continue"
228 #echo "INFO "$callistocount" RunCallisto.sh are running -> continue"
229 continue
230 fi
231 callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$night"_"$runnum"-calibration.log"
232 if ! [ -e $callistolog ]
233 then
234 rsync_aux_file $drivefilenewdaq2 $drivefile2
235 rsync_aux_file $mweatherfilenewdaq $mweatherfile
236 rsync_aux_file $ratesfilenewdaq $ratesfile
237 rsync_aux_file $tempfilenewdaq $tempfile
238 rsync_aux_file $humfilenewdaq $humfile
239 #printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
240 #echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
241 #`dirname $0`/RunCallisto.sh $sourcekey $seqfile &
242 printprocesslog "INFO starting RunCallisto.sh for "$seqfile
243 #echo "INFO starting RunCallisto.sh for "$seqfile
244 `dirname $0`/RunCallisto.sh $seqfile &
245 fi
246 continue
247 else
248 # skip a non-data run when it has not 1000 evts
249 # as this means probably an fad-loss
250 if [ $numevts -ne 1000 ]
251 then
252 printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$nmevts")"
253 continue
254 fi
255 fi
256 printprocesslog "DEBUG runrow "$runrow" (from variable) "
257 runrow=`cat $infopath/runrow.txt`
258 printprocesslog "DEBUG runrow "$runrow" (from file) "
259 runrow=$runrow$runnum$runtype$roi"_"
260 echo $runrow > $infopath/runrow.txt
261 if echo $runrow | grep -E '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_' >/dev/null
262 then
263 runrow2=`echo $runrow | grep -E -o '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'`
264 run1=`echo $runrow2 | cut -d_ -f1 | sed -e 's/drs-pedestal1024//g'`
265 run2=`echo $runrow2 | cut -d_ -f2 | sed -e 's/drs-gain1024//g'`
266 run3=`echo $runrow2 | cut -d_ -f3 | sed -e 's/drs-pedestal1024//g'`
267 run4=`echo $runrow2 | cut -d_ -f4 | sed -e 's/drs-pedestal1024//g'`
268 run5=`echo $runrow2 | cut -d_ -f5 | sed -e 's/drs-time1024//g'`
269 run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
270 run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
271 seqfile=$seqpath/${night}_${run1}.drs.seq
272 printprocesslog "INFO write drs-seq "$seqfile
273 echo "# written by automatic analysis in LP" > $seqfile
274 echo "" >> $seqfile
275 echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
276 echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
277 echo "" >> $seqfile
278 echo "CalRuns: "$run2 >> $seqfile
279 echo "PedRuns: "$run6" "$run7 >> $seqfile
280 echo "DatRuns: "$run5 >> $seqfile
281 echo "DrsRuns: "$run1" "$run3" "$run4 >> $seqfile
282 echo "DrsFiles: "$run3" "$run6 >> $seqfile
283 echo "" >> $seqfile
284 echo "#DrsFile: "$run6 >> $seqfile
285 echo "" >> $seqfile
286 echo "" > $infopath/runrow.txt
287 drsseq=$run1
288 drsfile=$run6
289 fi
290 if echo $runrow | grep -E '[0-9]{3}pedestal300_[0-9]{3}light-pulser-ext300_' >/dev/null
291 then
292 echo "" > $infopath/runrow.txt
293 fi
294 fi
295 done
296 printprocesslog "INFO status after loop: "$callistocount" callistos running, "$numdataruns" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
297
298 # get new file lists
299 printprocesslog "INFO get new file lists for "$datepath
300 files=( `find $rawpathnewdaq -type f | sort` )
301 fileslocal=( `find $rawpath -type f | sort` )
302 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
303 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
304 printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
305
306 # wait and get new file lists
307 update=
308 if [ ${#fileslocal[@]} -eq ${#files[@]} ]
309 then
310 printprocesslog "INFO wait 60 seconds."
311 sleep 60
312 #echo "sleep 60..."
313 printprocesslog "INFO get new file lists for "$datepath
314 files=( `find $rawpathnewdaq -type f | sort` )
315 fileslocal=( `find $rawpath -type f | sort` )
316 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
317 fi
318 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
319 printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
320done
321
322
323
324
Note: See TracBrowser for help on using the repository browser.