source: trunk/DataCheck/QuickLook/Step1.sh@ 17066

Last change on this file since 17066 was 17003, checked in by Daniela Dorner, 11 years ago
added directory for temporary rsync files
  • Property svn:executable set to *
File size: 14.9 KB
Line 
1#!/bin/bash
2#
3source `dirname $0`/../Sourcefile.sh
4printprocesslog "INFO starting $0"
5
6# get date (before 18h there is no new data to be processed)
7datepath=`date --date="-19HOUR" +%Y/%m/%d`
8date=`date --date="-19HOUR" +%Y%m%d`
9#datepath="2013/07/13"
10#date="20130713"
11printprocesslog "INFO processing "$datepath
12night=`echo $datepath | sed -e 's/\///g'`
13
14auxpathnewdaq=/newdaq/aux/$datepath
15# create aux directory on daq, if not yet there
16auxpath=/loc_data/aux/$datepath
17makedir $auxpath
18# create path for info files needed for analysis
19infopath=$anapath/info/$datepath
20makedir $infopath
21echo "" > $infopath/runrow.txt
22# create path for sequence files
23seqpath=$anapath/sequences/$datepath
24makedir $seqpath
25rawpathnewdaq=/newdaq/raw/$datepath
26rawpath=/loc_data/raw/$datepath
27
28# needed auxiliary files:
29# drive file with information about current source position
30drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
31drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
32# drive file with information about tracking position
33drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
34drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
35# file with magic weather information
36mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
37mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
38# file with trigger rates
39ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
40ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
41# file with trigger rates
42tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
43tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
44# file with trigger rates
45humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
46humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits
47
48function rsync_aux_file()
49{
50 if ls $1 >/dev/null 2>&1
51 then
52 printprocesslog "INFO rsync "$1
53 #if ! rsync -av --stats $1 $2
54 #if ! rsync -av $1 $2
55 if ! rsync -a -T $rsynctempdir $1 $2
56 then
57 printprocesslog "WARN rsync of "$1" failed."
58 fi
59 else
60 printprocesslog "WARN "$1" missing."
61 fi
62}
63
64function check_daq()
65{
66 diskusage=( `df -P /raid10 | grep raid10 ` )
67 # check if more than 700 GB are left on /loc_data
68 if [ ${diskusage[3]} -lt $disklimitdaq ]
69 then
70 echo "WARN less than 700 left on /raid10 on node "$HOSTNAME
71 printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME
72 df -h /raid10
73 finish
74 fi
75}
76
77check_daq
78
79printprocesslog "INFO get lists of raw files on newdaq and daq"
80files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
81if [ ${#files[@]} -eq 0 ]
82then
83 printprocesslog "INFO no raw files available yet for "$datepath
84 finish
85fi
86fileslocal=( `find $rawpath -type f | sort` )
87callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
88numdataruns=0
89
90# create raw directory on daq, if not yet there
91makedir $rawpath
92
93#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
94printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
95
96while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ]
97do
98 # only continue with script
99 # when there is more than 10% space on daq
100 source `dirname $0`/../Sourcefile.sh
101 check_daq
102
103 numdataruns=0
104 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
105 printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
106
107 rsync_aux_file $drivefilenewdaq $drivefile
108
109 # files on newdaq
110 for file in ${files[@]}
111 do
112 printprocesslog "processing "$file
113 localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
114 #echo "processing "$file" "$localfile
115
116 if [ "`echo $file | grep -o drs`" == "drs" ]
117 then
118 nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
119 tstop=`$factpath/fitsdump -h $nondrsfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
120 else
121 tstop=`$factpath/fitsdump -h $file 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
122 fi
123 if [ "$tstop" == "0" ]
124 then
125 printprocesslog "WARN "$file" not yet closed."
126 fileaccessed=`find $file -amin -30`
127 if ! [ "$fileaccessed" == "" ]
128 then
129 printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
130 continue
131 else
132 printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
133 fileerror="yes"
134 fi
135 fi
136
137 source `dirname $0`/../Sourcefile.sh
138 if ! ls $localfile >/dev/null 2>&1
139 then
140 #if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
141 if ! rsync -a -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
142 then
143 printprocesslog "ERROR something went wrong with rsync of "$file
144 rm $localfile
145 continue
146 fi
147 printprocesslog "INFO "$file" rsynced successfully."
148 fi
149
150
151 if [ "`echo $localfile | grep -o drs`" != "drs" ]
152 then
153 runtype=`$factpath/fitsdump -h $localfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
154 runnum=`echo $localfile | cut -d_ -f3 | cut -d. -f1`
155 roi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
156 numevts=`$factpath/fitsdump -h $file 2>/dev/null | grep Events | grep -E -o '[0-9]+'`
157 printprocesslog "DEBUG runnum "$runnum" runtype "$runtype" roi "$roi" numevts "$numevts
158 if [ "$runtype" == "drs-time-upshifted" ]
159 then
160 printprocesslog "INFO file "$file" has runtype drs-time-upshifted -> continue "
161 continue
162 fi
163 #echo $runtype" "$runnum
164 if [ "$runtype" == "data" ]
165 then
166 if [ "$fileerror" = "yes" ]
167 then
168 printprocesslog "INFO do not further process corrupted file "$localfile
169 fileerror=
170 continue
171 fi
172 seqfile=$seqpath/${night}_${runnum}.seq
173 printprocesslog "INFO write data-seq "$seqfile
174 echo "# written by automatic analysis in LP" >$seqfile
175 echo "" >> $seqfile
176 echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
177 echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
178 echo "" >> $seqfile
179 echo "DrsSequence: "$drsseq >> $seqfile
180 echo "" >> $seqfile
181# echo $runrow" CalRuns"
182# echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'
183 echo "CalRuns: "`echo $runrow | grep -E -o '[0-9]{3}light-pulser-ext300' | sed -e 's/light-pulser-ext300//g'` >> $seqfile
184 echo "PedRuns: "`echo $runrow | grep -E -o '[0-9]{3}pedestal300' | sed -e 's/pedestal300//g'` >> $seqfile
185 echo "DatRuns: "$runnum >> $seqfile
186 echo "" >> $seqfile
187 echo "DrsFiles: "$drsfile >> $seqfile
188 echo "" >> $seqfile
189 echo "#DrsFile: "$drsfile >> $seqfile
190 echo "" >> $seqfile
191
192# tstopi=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
193# tstopf=`$factpath/fitsdump -h $localfile 2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
194# tstop=${tstopi}${tstopf}
195# coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop} 2>/dev/null | tail -1 2>&1` )
196# if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
197# then
198# printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
199# #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
200# continue
201# fi
202# if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
203# then
204# printprocesslog "WARN coordinates "${coordinates[@]}
205# #echo "WARN coordinates "${coordinates[@]}
206# continue
207# fi
208# printprocesslog "DEBUG coordinates "${coordinates[@]}
209# query="SELECT fSourceKEY FROM scheduling.source WHERE "
210# query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
211# query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
212# sourcekey=`sendquery`
213# if [ "$sourcekey" == "" ]
214# then
215# printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
216# fi
217
218 printprocesslog "INFO counting callisto logs and data files +1."
219 # get number of callisto logs
220 callistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
221 # count data runs
222 numdataruns=`echo " $numdataruns + 1 " | bc -l`
223 #echo "numdata +1"
224
225 #echo "cal: "$callistocount" numdat: "$numdataruns" numcallog: "${#callistofiles[@]}
226 printprocesslog "INFO running callistos: "$callistocount" #data-runs: "$numdataruns" #callisto-logs: "${#callistofiles[@]}
227 # do not overload system in case of a lot of files to be processed
228 # callistocount is set in setup.fact.lp.data
229 if [ $callistocount -ge $numcallistos ]
230 then
231 printprocesslog "INFO "$callistocount" RunCallisto.sh are running -> continue"
232 #echo "INFO "$callistocount" RunCallisto.sh are running -> continue"
233 continue
234 fi
235 callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$night"_"$runnum"-calibration.log"
236 if ! [ -e $callistolog ]
237 then
238 rsync_aux_file $drivefilenewdaq2 $drivefile2
239 rsync_aux_file $mweatherfilenewdaq $mweatherfile
240 rsync_aux_file $ratesfilenewdaq $ratesfile
241 rsync_aux_file $tempfilenewdaq $tempfile
242 rsync_aux_file $humfilenewdaq $humfile
243 #printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
244 #echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
245 #`dirname $0`/RunCallisto.sh $sourcekey $seqfile &
246 printprocesslog "INFO starting RunCallisto.sh for "$seqfile
247 #echo "INFO starting RunCallisto.sh for "$seqfile
248 `dirname $0`/RunCallisto.sh $seqfile &
249 fi
250 continue
251 else
252 # skip a non-data run when it has not 1000 evts
253 # as this means probably an fad-loss
254 if [ $numevts -ne 1000 ]
255 then
256 printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$nmevts")"
257 continue
258 fi
259 fi
260 printprocesslog "DEBUG runrow "$runrow" (from variable) "
261 runrow=`cat $infopath/runrow.txt`
262 printprocesslog "DEBUG runrow "$runrow" (from file) "
263 runrow=$runrow$runnum$runtype$roi"_"
264 echo $runrow > $infopath/runrow.txt
265 if echo $runrow | grep -E '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_' >/dev/null
266 then
267 runrow2=`echo $runrow | grep -E -o '[0-9]{3}drs-pedestal1024_[0-9]{3}drs-gain1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-pedestal1024_[0-9]{3}drs-time1024_[0-9]{3}pedestal300_[0-9]{3}pedestal300_'`
268 run1=`echo $runrow2 | cut -d_ -f1 | sed -e 's/drs-pedestal1024//g'`
269 run2=`echo $runrow2 | cut -d_ -f2 | sed -e 's/drs-gain1024//g'`
270 run3=`echo $runrow2 | cut -d_ -f3 | sed -e 's/drs-pedestal1024//g'`
271 run4=`echo $runrow2 | cut -d_ -f4 | sed -e 's/drs-pedestal1024//g'`
272 run5=`echo $runrow2 | cut -d_ -f5 | sed -e 's/drs-time1024//g'`
273 run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
274 run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
275 seqfile=$seqpath/${night}_${run1}.drs.seq
276 printprocesslog "INFO write drs-seq "$seqfile
277 echo "# written by automatic analysis in LP" > $seqfile
278 echo "" >> $seqfile
279 echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
280 echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
281 echo "" >> $seqfile
282 echo "CalRuns: "$run2 >> $seqfile
283 echo "PedRuns: "$run6" "$run7 >> $seqfile
284 echo "DatRuns: "$run5 >> $seqfile
285 echo "DrsRuns: "$run1" "$run3" "$run4 >> $seqfile
286 echo "DrsFiles: "$run3" "$run6 >> $seqfile
287 echo "" >> $seqfile
288 echo "#DrsFile: "$run6 >> $seqfile
289 echo "" >> $seqfile
290 echo "" > $infopath/runrow.txt
291 drsseq=$run1
292 drsfile=$run6
293 fi
294 if echo $runrow | grep -E '[0-9]{3}pedestal300_[0-9]{3}light-pulser-ext300_' >/dev/null
295 then
296 echo "" > $infopath/runrow.txt
297 fi
298 fi
299 done
300 printprocesslog "INFO status after loop: "$callistocount" callistos running, "$numdataruns" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
301
302 # get new file lists
303 printprocesslog "INFO get new file lists for "$datepath
304 files=( `find $rawpathnewdaq -type f | sort` )
305 fileslocal=( `find $rawpath -type f | sort` )
306 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
307 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
308 printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
309
310 # wait and get new file lists
311 update=
312 if [ ${#fileslocal[@]} -eq ${#files[@]} ]
313 then
314 printprocesslog "INFO wait 60 seconds."
315 sleep 60
316 #echo "sleep 60..."
317 printprocesslog "INFO get new file lists for "$datepath
318 files=( `find $rawpathnewdaq -type f | sort` )
319 fileslocal=( `find $rawpath -type f | sort` )
320 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
321 fi
322 #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
323 printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
324done
325
326
327
328
Note: See TracBrowser for help on using the repository browser.