source: trunk/DataCheck/QuickLook/Step1.sh @ 19474

Last change on this file since 19474 was 19474, checked in by dorner, 4 months ago
implemented possibility to skip magic-weather info
  • Property svn:executable set to *
File size: 17.9 KB
Line 
1#!/bin/bash
2#
3
4# to use script with /data1 instead of /scratch
5#  (e.g. when /scratch is full)
6# export data1=yes
7# before executing the script
8# preferably ZipRawData.sh has processed at this point all
9#   files available already on /scratch
10#
11# to have QLA running if MAGIC weather station is not available:
12# export mweather=no
13
14# remarks:
15# move fileerror check to main-loop?
16
17source `dirname $0`/../Sourcefile.sh
18printprocesslog "INFO starting $0"
19
20# get date (before 18h there is no new data to be processed)
21if [ "$certaindate" != "" ]
22then
23   checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
24   if [ "$checkstring" = "" ]
25   then
26      echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
27      finish
28   fi
29   datepath=$certaindate
30else
31   # this has to be coordinated with the time when
32   #   ClearNewdata.sh is running
33   datepath=`date --date="-18HOUR" +%Y/%m/%d`
34fi
35date=`echo $datepath | sed -e 's/\///g'`
36printprocesslog "INFO processing "$datepath
37
38auxpath=/loc_data/aux/$datepath
39makedir $auxpath >/dev/null
40# create path for info files needed for analysis
41infopath=$anapath/info/$datepath
42makedir $infopath >/dev/null
43echo "" > $infopath/runrow.txt
44# create path for callisto output
45calpath=$anapath/callisto/$datepath
46makedir $calpath >/dev/null
47rawpathnewdaq=/newdaq/raw/$datepath
48if [ "$data1" = "yes" ]
49then 
50   rawpath=/data1/raw/$datepath
51   rsynctempdir=/data1/rsync_tmp
52   printprocesslog "INFO using "$rawpath" for processing"
53else 
54   rawpath=/scratch/raw/$datepath
55   rsynctempdir=/scratch/rsync_tmp
56fi
57if ! [ -d $rsynctempdir ]
58then
59   mkdir $rsynctempdir
60fi
61
62
63
64# needed auxiliary files:
65#   drive file with information about current source position
66drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
67#   drive file with information tracking position
68drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
69#   file with magic weather information
70mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
71#   file with trigger rates
72ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
73#   file with trigger rates
74tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
75#   file with trigger rates
76humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
77
78function rsync_aux_file()
79{
80   if check_file_avail $1
81   then 
82      printprocesslog "INFO rsync "$1
83      if ! rsync -a -T $rsynctempdir newdaq:$1 $1
84      then 
85         printprocesslog "WARN rsync of "$1" failed."
86      fi
87   else
88       continue
89   fi
90}
91
92function check_disks()
93{
94   # at least 5% free disk on /data1
95   diskusage=( `df -P /data1 | grep data1 ` )
96   if [ ${diskusage[3]} -lt $disklimitnewdata2 ] 
97   then
98      echo "WARN less than 5% left on /data1 on node "$HOSTNAME
99      printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME
100      df -h /data1
101      finish
102   fi
103   
104   # if /data1 is used for processing, /scratch doesn't need to be checked
105   if [ "$data1" = "yes" ]
106   then
107      return
108   fi
109   
110   # at least 10% free disk on /scratch
111   diskusage=( `df -P /scratch | grep scratch ` )
112   printprocesslog "DEBUG dist /scratch "${diskusage[3]}
113   if [ ${diskusage[3]} -lt $disklimitnewdata ] 
114   then
115      echo "WARN less than 10% left on /scratch on node "$HOSTNAME
116      printprocesslog "WARN less than 10% left on /scratch on node "$HOSTNAME
117      df -h /scratch
118      finish
119   fi
120}
121
122check_disks
123
124# getting lists of files
125printprocesslog "INFO get lists of raw files on newdaq and daq"
126#files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
127files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
128if [ ${#files[@]} -eq 0 ]
129then
130   printprocesslog "INFO no raw files available yet for "$datepath
131   finish
132fi
133fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
134callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
135
136# get number of dataruns from DB
137query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
138numdataruns=`sendquery`
139query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6"
140numlpruns=`sendquery`
141query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300"
142numpedruns=`sendquery`
143query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5"
144numdrstime=`sendquery`
145numpedruns=0
146#numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l`
147numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l`
148
149# create raw directory on daq, if not yet there
150makedir $rawpath >/dev/null
151
152#echo "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
153printprocesslog "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
154
155while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
156do
157   # only continue with script
158   #  when there is more than 10% space on daq
159   source `dirname $0`/../Sourcefile.sh
160   
161   numcalibrated=0
162   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
163   printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
164
165   #rsync_aux_file $drivefile
166   #rsync_aux_file $drivefile2
167   
168   # files on newdaq
169   for file in ${files[@]}
170   do
171      # check if still enough diskspace for transfer
172      check_disks
173      if [ "$certaindate" != "" ]
174      then
175         echo "processing "$file
176      fi
177      printprocesslog "processing "$file
178      if [ "$data1" = "yes" ]
179      then 
180         localfile=`echo $file | sed -e 's/newdaq/data1/'`
181      else
182         localfile=`echo $file | sed -e 's/newdaq/scratch/'`
183      fi
184      #localfile=$file
185
186      source `dirname $0`/../Sourcefile.sh
187
188      # check if file is already transferred
189      if ! ls $localfile >/dev/null 2>&1 
190      then
191         # check if it is drs-file
192         #   get stop time from raw-file
193         if [ "`echo $file | grep -o drs`" == "drs" ]
194         then
195            nondrs=`basename $file | sed -e 's/[.]drs//g'`
196            nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null `
197            tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
198         else
199            tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
200         fi
201         # when stop time is 0, file is not closed
202         # when an error is returned the tstop is empty
203         if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
204         then
205            printprocesslog "DEBUG "$file" not yet closed."
206            # if a file is not closed and not touched for 30 minutes,
207            #    it is assumed corrupted and still transferred
208            fileaccessed=`find $file -amin -30 2>/dev/null `
209            if ! [ "$fileaccessed" == "" ]
210            then
211               printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
212               continue
213            else
214               printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
215               fileerror="yes"
216            fi
217         fi
218
219         # rsync
220         #if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile
221         if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
222         then
223            printprocesslog "ERROR something went wrong with rsync of "$file
224            rm $localfile
225            continue
226         fi
227         printprocesslog "INFO "$file" rsynced successfully."
228      fi
229
230      # for .drs.fits files no further treatment needed
231      if [ "`echo $localfile | grep -o drs`" == "drs" ]
232      then
233         continue
234      fi
235     
236#      # temporary check (needed to run on newdaq)
237#      if [ "`echo $file | grep -o drs`" == "drs" ]
238#      then
239#         nondrs=`basename $file | sed -e 's/[.]drs//g'`
240#         nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null `
241#         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
242#      else
243#         tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
244#      fi
245#      # when stop time is 0, file is not closed
246#      # when an error is returned the tstop is empty
247#      if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
248#      then
249#         printprocesslog "WARN "$file" not yet closed. -> continue (temporary check)"
250#         continue
251#      fi
252#      # end temporary check
253     
254      # treat other files (.fits.fz)
255      runtype=`$factpath/fitsdump -h $localfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
256      if [ "$runtype" != "data" ]
257      then
258         # skip a non-data run when it has not 1000 evts
259         #   as this means probably an fad-loss
260         #   and these runs are repeated in that case
261         numevts=`$factpath/fitsdump -h $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
262         if [ "$numevts" == "" ]
263         then
264            printprocesslog "WARN could not get number of events from file "$file" -> continue "
265            #echo "WARN could not get number of events from file "$file" -> continue "
266            continue
267         fi
268         if [ $numevts -ne 1000 ]
269         then
270            printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")"
271            continue
272         fi
273      fi
274
275      # get run number
276      runnum=`basename $localfile | cut -d_ -f2 | cut -d. -f1`
277     
278      # what is needed to process the different runs?
279      #   P: run#(P), run#(drs-file)
280      #   C: run#(C), run#(drs-file), run#(drs-time)
281      #   D: run#(D), run#(drs-file), run#(drs-time), ?
282      # what is drs-file? pedestal, roi300, has drs.fits
283      callistolog=$calpath"/"$date"_"$runnum"-calibration.log"
284      case $runtype in
285         data) # treat D-runs
286            if [ "$fileerror" = "yes" ]
287            then
288               printprocesslog "INFO do not further process corrupted file "$localfile
289               fileerror=
290               continue
291            fi
292           
293            # some accounting
294            printprocesslog "DEBUG counting callisto logs and set data files +1."
295            # get number of callisto logs
296            runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
297            # count runs to be calibrated
298            numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
299            printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]}
300           
301            # do not overload system in case of a lot of files to be processed
302            #  numruncallistos is set in setup.fact.lp.data
303            if [ $runcallistocount -ge $numruncallistos ]
304            then
305               printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue"
306               continue
307            fi
308           
309            # starting calibration
310            if ! [ -e $callistolog ]
311            then
312               rsync_aux_file $drivefile
313               rsync_aux_file $drivefile2
314               if [ "$mweather" != "no" ]
315               then 
316                 rsync_aux_file $mweatherfile
317               fi
318               rsync_aux_file $ratesfile
319               rsync_aux_file $tempfile
320               rsync_aux_file  $humfile
321               
322               calfile=$calpath"/"$date"_"$runnum"_C.root"
323               printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" outpath "$outpath" calfile "$calfile
324               `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $calpath $calfile &
325            fi
326            continue
327            ;;
328         pedestal) # treat P-runs
329            roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
330            if [ $roi -eq 300 ]
331            then
332               # check drs-file
333               drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
334               if [ -e $drsfile ]
335               then
336                  # set name of drs-file
337                  drscalib=$drsfile
338                  continue
339               #else
340                  # not needed for QLA
341                  #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
342                  #if ! [ -e $callistolog ]
343                  #then
344                  #   pedfile=$calpath"/"$date"_"$runnum"-pedestal.root"
345                  #   # count runs to be calibrated
346                  #   printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
347                  #   echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
348                  #  `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile &
349                  #fi
350               fi
351            fi
352            ;;
353         light-pulser-ext) # treat C-runs
354            # do lp-treatment -> not needed for QLA
355            #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root"
356            #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
357            #if ! [ -e $callistolog ]
358            #then
359            #   if [ -e $drstime ]
360            #   then
361            #      # count runs to be calibrated
362            #      printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
363            #      echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
364            #      `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile &
365            #   fi
366            #fi
367            ;;
368         drs-time) # treat drs-time runs
369            # do drs-timing calibration
370            drstime=$calpath"/"$date"_"$runnum"-drstime.root"
371            # starting calibration
372            numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
373            if ! [ -e $callistolog ]
374            then
375               # count runs to be calibrated
376               printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
377               #echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
378               `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime &
379            fi
380            ;;
381         drs-pedestal) # get drs-pedestal
382            roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
383            drs=`$factpath/fitsdump -h $localfile  2>/dev/null | grep DRSCALIB | grep -E -o " T " `
384            if [ $roi -eq 1024 ] && [ "$drs" == " T " ]
385            then
386               drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
387            fi
388            ;;
389         *) # other runs
390            printprocesslog "INFO file "$file" has runtype "$runtype" -> continue "
391            continue
392            ;;
393      esac
394   done
395   printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
396   
397   # get new file lists
398   printprocesslog "INFO get new file lists for "$datepath
399   #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
400   files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
401   fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
402   callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
403   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
404   printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
405
406   # wait and get new file lists
407   update=
408   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
409   then 
410      printprocesslog "INFO wait 60 seconds."
411      sleep 60
412      printprocesslog "INFO get new file lists for "$datepath
413      #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
414      files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
415      fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
416      callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
417   fi
418   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
419   printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
420   sleep 30
421   printprocesslog "INFO sleep 30"
422done
423
424
425finish
Note: See TracBrowser for help on using the repository browser.