source: trunk/DataCheck/QuickLook/Step1.sh@ 15658

Last change on this file since 15658 was 15591, checked in by Daniela Dorner, 11 years ago
moved diskusage check to function and implemented it also in loop
  • Property svn:executable set to *
File size: 14.7 KB
Line 
1#!/bin/bash
2#
3source `dirname $0`/../Sourcefile.sh
4printprocesslog "INFO starting $0"
5
6# get date (before 18h there is no new data to be processed)
7datepath=`date --date="-19HOUR" +%Y/%m/%d`
8date=`date --date="-19HOUR" +%Y%m%d`
9#datepath="2013/04/17"
10#date="20130417"
11printprocesslog "INFO processing "$datepath
12night=`echo $datepath | sed -e 's/\///g'`
13
14auxpathnewdaq=/newdaq/aux/$datepath
15# create aux directory on daq, if not yet there
16auxpath=/loc_data/aux/$datepath
17makedir $auxpath
18# create path for info files needed for analysis
19infopath=$anapath/info/$datepath
20makedir $infopath
21echo "" > $infopath/runrow.txt
22# create path for sequence files
23seqpath=$anapath/sequences/$datepath
24makedir $seqpath
25rawpathnewdaq=/newdaq/raw/$datepath
26rawpath=/loc_data/raw/$datepath
27
28# needed auxiliary files:
29# drive file with information about current source position
30drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
31drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
32# drive file with information about tracking position
33drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
34drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
35# file with magic weather information
36mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
37mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
38# file with trigger rates
39ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
40ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
41# file with trigger rates
42tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
43tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
44# file with trigger rates
45humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
46humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits
47
48function rsync_aux_file()
49{
50 if ls $1 >/dev/null 2>&1
51 then
52 printprocesslog "INFO rsync "$1
53 #if ! rsync -av --stats $1 $2
54 #if ! rsync -av $1 $2
55 if ! rsync -a $1 $2
56 then
57 printprocesslog "WARN rsync of "$1" failed."
58 fi
59 else
60 printprocesslog "WARN "$1" missing."
61 fi
62}
63
64function check_daq()
65{
66 diskusage=( `df -P /raid10 | grep raid10 ` )
67 # check if more than 700 GB are left on /loc_data
68 if [ ${diskusage[3]} -lt $disklimitdaq ]
69 then
70 echo "WARN less than 700 left on /raid10 on node "$HOSTNAME
71 printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME
72 df -h /raid10
73 finish
74 fi
75}
76
77check_daq
78
79printprocesslog "INFO get lists of raw files on newdaq and daq"
80files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
81if [ ${#files[@]} -eq 0 ]
82then
83 printprocesslog "INFO no raw files available yet for "$datepath
84 finish
85fi
86fileslocal=( `find $rawpath -type f | sort` )
87callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
88numdataruns=0
89
90# create raw directory on daq, if not yet there
91makedir $rawpath
92
93#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
94printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
95
96while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ]
97do
98 # only continue with script
99 # when there is more than 10% space on daq
100 source `dirname $0`/../Sourcefile.sh
101 check_daq
102
103 numdataruns=0
104 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
105 printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
106
107 rsync_aux_file $drivefilenewdaq $drivefile
108
109 # files on newdaq
110 for file in ${files[@]}
111 do
112 printprocesslog "processing "$file
113 localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
114 #echo "processing "$file" "$localfile
115
116 if [ "`echo $file | grep -o drs`" == "drs" ]
117 then
118 nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
119 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
120 else
121 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
122 fi
123 if [ "$tstop" == "0" ]
124 then
125 printprocesslog "WARN "$file" not yet closed."
126 fileaccessed=`find $file -amin -30`
127 if ! [ "$fileaccessed" == "" ]
128 then
129 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
130 continue
131 else
132 printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
133 fileerror="yes"
134 fi
135 fi
136
137 source `dirname $0`/../Sourcefile.sh
138 if ! ls $localfile >/dev/null 2>&1
139 then
140 #if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
141 if ! rsync -a --bwlimit=$bwlimit $file $localfile
142 then
143 printprocesslog "ERROR something went wrong with rsync of "$file
144 rm $localfile
145 continue
146 fi
147 printprocesslog "INFO "$file" rsynced successfully."
148 fi
149
150
151 if [ "`echo $localfile | grep -o drs`" != "drs" ]
152 then
153 runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
154 runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
155 roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
156 numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
157 printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi" numevts "$numevts
158 if [ "$runtype" == "drs-time-upshifted" ]
159 then
160 printprocesslog "INFO file "$file" has runtype drs-time-upshifted -> continue "
161 continue
162 fi
163 if [ "$runtype" == "data" ]
164 then
165 if [ "$fileerror" = "yes" ]
166 then
167 printprocesslog "INFO do not further process corrupted file "$localfile
168 fileerror=
169 continue
170 fi
171 seqfile=$seqpath/${night}_${runnum}.seq
172 printprocesslog "INFO write data-seq "$seqfile
173 echo "# written by automatic analysis in LP" >$seqfile
174 echo "" >> $seqfile
175 echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
176 echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
177 echo "" >> $seqfile
178 echo "DrsSequence: "$drsseq >> $seqfile
179 echo "" >> $seqfile
180 echo "CalRuns: "`echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'` >> $seqfile
181 echo "PedRuns: "`echo $runrow | grep -E -o '[0-9]{3}pedestal300' | sed -e 's/pedestal300//g'` >> $seqfile
182 echo "DatRuns: "$runnum >> $seqfile
183 echo "" >> $seqfile
184 echo "DrsFiles: "$drsfile >> $seqfile
185 echo "" >> $seqfile
186 echo "#DrsFile: "$drsfile >> $seqfile
187 echo "" >> $seqfile
188
189# tstopi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
190# tstopf=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
191# tstop=${tstopi}${tstopf}
192# coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop} 2>/dev/null | tail -1 2>&1` )
193# if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
194# then
195# printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
196# #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
197# continue
198# fi
199# if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
200# then
201# printprocesslog "WARN coordinates "${coordinates[@]}
202# #echo "WARN coordinates "${coordinates[@]}
203# continue
204# fi
205# printprocesslog "DEBUG coordinates "${coordinates[@]}
206# query="SELECT fSourceKEY FROM scheduling.source WHERE "
207# query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
208# query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
209# sourcekey=`sendquery`
210# if [ "$sourcekey" == "" ]
211# then
212# printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
213# fi
214
215 printprocesslog "INFO counting callisto logs and data files +1."
216 # get number of callisto logs
217 callistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
218 # count data runs
219 numdataruns=`echo " $numdataruns + 1 " | bc -l`
220 #echo "numdata +1"
221
222 #echo "cal: "$callistocount" numdat: "$numdataruns" numcallog: "${#callistofiles[@]}
223 printprocesslog "INFO running callistos: "$callistocount" #data-runs: "$numdataruns" #callisto-logs: "${#callistofiles[@]}
224 # do not overload system in case of a lot of files to be processed
225 # callistocount is set in setup.fact.lp.data
226 if [ $callistocount -ge $numcallistos ]
227 then
228 printprocesslog "INFO "$callistocount" RunCallisto.sh are running -> continue"
229 #echo "INFO "$callistocount" RunCallisto.sh are running -> continue"
230 continue
231 fi
232 callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$night"_"$runnum"-calibration.log"
233 if ! [ -e $callistolog ]
234 then
235 rsync_aux_file $drivefilenewdaq2 $drivefile2
236 rsync_aux_file $mweatherfilenewdaq $mweatherfile
237 rsync_aux_file $ratesfilenewdaq $ratesfile
238 rsync_aux_file $tempfilenewdaq $tempfile
239 rsync_aux_file $humfilenewdaq $humfile
240 #printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
241 #echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
242 #`dirname $0`/RunCallisto.sh $sourcekey $seqfile &
243 printprocesslog "INFO starting RunCallisto.sh for "$seqfile
244 #echo "INFO starting RunCallisto.sh for "$seqfile
245 `dirname $0`/RunCallisto.sh $seqfile &
246 fi
247 continue
248 else
249 # skip a non-data run when it has not 1000 evts
250 # as this means probably an fad-loss
251 if [ $numevts -ne 1000 ]
252 then
253 printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$nmevts")"
254 continue
255 fi
256 fi
257 printprocesslog "DEBUG runrow "$runrow" (from variable) "
258 runrow=`cat $infopath/runrow.txt`
259 printprocesslog "DEBUG runrow "$runrow" (from file) "
260 runrow=$runrow$runnum$runtype$roi"_"
261 echo $runrow > $infopath/runrow.txt
262 if echo $runrow | grep -E '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_' >/dev/null
263 then
264 runrow2=`echo $runrow | grep -E -o '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'`
265 run1=`echo $runrow2 | cut -d_ -f1 | sed -e 's/drs-pedestal1024//g'`
266 run2=`echo $runrow2 | cut -d_ -f2 | sed -e 's/drs-gain1024//g'`
267 run3=`echo $runrow2 | cut -d_ -f3 | sed -e 's/drs-pedestal1024//g'`
268 run4=`echo $runrow2 | cut -d_ -f4 | sed -e 's/drs-pedestal1024//g'`
269 run5=`echo $runrow2 | cut -d_ -f5 | sed -e 's/drs-time1024//g'`
270 run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
271 run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
272 seqfile=$seqpath/${night}_${run1}.drs.seq
273 printprocesslog "INFO write drs-seq "$seqfile
274 echo "# written by automatic analysis in LP" > $seqfile
275 echo "" >> $seqfile
276 echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
277 echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
278 echo "" >> $seqfile
279 echo "CalRuns: "$run2 >> $seqfile
280 echo "PedRuns: "$run6" "$run7 >> $seqfile
281 echo "DatRuns: "$run5 >> $seqfile
282 echo "DrsRuns: "$run1" "$run3" "$run4 >> $seqfile
283 echo "DrsFiles: "$run3" "$run6 >> $seqfile
284 echo "" >> $seqfile
285 echo "#DrsFile: "$run6 >> $seqfile
286 echo "" >> $seqfile
287 echo "" > $infopath/runrow.txt
288 drsseq=$run1
289 drsfile=$run6
290 fi
291 if echo $runrow | grep -E '[0-9]{3}pedestal300_[0-9]{3}light-pulser-ext300_' >/dev/null
292 then
293 echo "" > $infopath/runrow.txt
294 fi
295 fi
296 done
297 printprocesslog "INFO status after loop: "$callistocount" callistos running, "$numdataruns" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
298
299 # get new file lists
300 printprocesslog "INFO get new file lists for "$datepath
301 files=( `find $rawpathnewdaq -type f | sort` )
302 fileslocal=( `find $rawpath -type f | sort` )
303 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
304 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
305 printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
306
307 # wait and get new file lists
308 update=
309 if [ ${#fileslocal[@]} -eq ${#files[@]} ]
310 then
311 printprocesslog "INFO wait 60 seconds."
312 sleep 60
313 #echo "sleep 60..."
314 printprocesslog "INFO get new file lists for "$datepath
315 files=( `find $rawpathnewdaq -type f | sort` )
316 fileslocal=( `find $rawpath -type f | sort` )
317 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
318 fi
319 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
320 printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
321done
322
323
324
325
Note: See TracBrowser for help on using the repository browser.