source: trunk/DataCheck/QuickLook/Step1.sh @ 18869

Last change on this file since 18869 was 18869, checked in by dorner, 2 years ago
fixed bug and implemented option to use /data1 instead of /scratch
  • Property svn:executable set to *
File size: 17.7 KB
Line 
1#!/bin/bash
2#
3
4# to use script with /data1 instead of /scratch
5#  (e.g. when /scratch is full)
6# export data1=yes
7# before executing the script
8# preferably ZipRawData.sh has processed at this point all
9#   files available already on /scratch
10
11# remarks:
12# move fileerror check to main-loop?
13
14source `dirname $0`/../Sourcefile.sh
15printprocesslog "INFO starting $0"
16
17# get date (before 18h there is no new data to be processed)
18if [ "$certaindate" != "" ]
19then
20   checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
21   if [ "$checkstring" = "" ]
22   then
23      echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
24      finish
25   fi
26   datepath=$certaindate
27else
28   # this has to be coordinated with the time when
29   #   ClearNewdata.sh is running
30   datepath=`date --date="-18HOUR" +%Y/%m/%d`
31fi
32date=`echo $datepath | sed -e 's/\///g'`
33printprocesslog "INFO processing "$datepath
34
35auxpath=/loc_data/aux/$datepath
36makedir $auxpath >/dev/null
37# create path for info files needed for analysis
38infopath=$anapath/info/$datepath
39makedir $infopath >/dev/null
40echo "" > $infopath/runrow.txt
41# create path for callisto output
42calpath=$anapath/callisto/$datepath
43makedir $calpath >/dev/null
44rawpathnewdaq=/newdaq/raw/$datepath
45if [ "$data1" = "yes" ]
46then 
47   rawpath=/data1/raw/$datepath
48   rsynctempdir=/data1/rsync_tmp
49   printprocesslog "INFO using "$rawpath" for processing"
50else 
51   rawpath=/scratch/raw/$datepath
52   rsynctempdir=/scratch/rsync_tmp
53fi
54if ! [ -d $rsynctempdir ]
55then
56   mkdir $rsynctempdir
57fi
58
59
60
61# needed auxiliary files:
62#   drive file with information about current source position
63drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
64#   drive file with information tracking position
65drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
66#   file with magic weather information
67mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
68#   file with trigger rates
69ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
70#   file with trigger rates
71tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
72#   file with trigger rates
73humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
74
75function rsync_aux_file()
76{
77   if check_file_avail $1
78   then 
79      printprocesslog "INFO rsync "$1
80      if ! rsync -a -T $rsynctempdir newdaq:$1 $1
81      then 
82         printprocesslog "WARN rsync of "$1" failed."
83      fi
84   fi
85}
86
87function check_disks()
88{
89   # at least 5% free disk on /data1
90   diskusage=( `df -P /data1 | grep data1 ` )
91   if [ ${diskusage[3]} -lt $disklimitnewdata2 ] 
92   then
93      echo "WARN less than 5% left on /data1 on node "$HOSTNAME
94      printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME
95      df -h /data1
96      finish
97   fi
98   
99   # if /data1 is used for processing, /scratch doesn't need to be checked
100   if [ "$data1" = "yes" ]
101   then
102      return
103   fi
104   
105   # at least 10% free disk on /scratch
106   diskusage=( `df -P /scratch | grep scratch ` )
107   if [ ${diskusage[3]} -lt $disklimitnewdata ] 
108   then
109      echo "WARN less than 10% left on /scratch on node "$HOSTNAME
110      printprocesslog "WARN less than 10% left on /scratch on node "$HOSTNAME
111      df -h /scratch
112      finish
113   fi
114}
115
116check_disks
117
118# getting lists of files
119printprocesslog "INFO get lists of raw files on newdaq and daq"
120#files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
121files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
122if [ ${#files[@]} -eq 0 ]
123then
124   printprocesslog "INFO no raw files available yet for "$datepath
125   finish
126fi
127fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
128callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
129
130# get number of dataruns from DB
131query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
132numdataruns=`sendquery`
133query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=6"
134numlpruns=`sendquery`
135query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=2 AND fHasDrsFile=1 AND fROI=300"
136numpedruns=`sendquery`
137query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=5"
138numdrstime=`sendquery`
139numpedruns=0
140#numcalibrated=`echo " $numdataruns + $numlpruns + $numpedruns + $numdrstime " | bc -l`
141numcalibrated=`echo " $numdataruns + $numdrstime " | bc -l`
142
143# create raw directory on daq, if not yet there
144makedir $rawpath >/dev/null
145
146#echo "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
147printprocesslog "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
148
149while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
150do
151   # only continue with script
152   #  when there is more than 10% space on daq
153   source `dirname $0`/../Sourcefile.sh
154   
155   numcalibrated=0
156   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
157   printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
158
159   rsync_aux_file $drivefile
160   rsync_aux_file $drivefile2
161   
162   # files on newdaq
163   for file in ${files[@]}
164   do
165      # check if still enough diskspace for transfer
166      check_disks
167      if [ "$certaindate" != "" ]
168      then
169         echo "processing "$file
170      fi
171      printprocesslog "processing "$file
172      if [ "$data1" = "yes" ]
173      then 
174         localfile=`echo $file | sed -e 's/newdaq/data1/'`
175      else
176         localfile=`echo $file | sed -e 's/newdaq/scratch/'`
177      fi
178      #localfile=$file
179
180      source `dirname $0`/../Sourcefile.sh
181
182      # check if file is already transferred
183      if ! ls $localfile >/dev/null 2>&1 
184      then
185         # check if it is drs-file
186         #   get stop time from raw-file
187         if [ "`echo $file | grep -o drs`" == "drs" ]
188         then
189            nondrs=`basename $file | sed -e 's/[.]drs//g'`
190            nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null `
191            tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
192         else
193            tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
194         fi
195         # when stop time is 0, file is not closed
196         # when an error is returned the tstop is empty
197         if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
198         then
199            printprocesslog "DEBUG "$file" not yet closed."
200            # if a file is not closed and not touched for 30 minutes,
201            #    it is assumed corrupted and still transferred
202            fileaccessed=`find $file -amin -30 2>/dev/null `
203            if ! [ "$fileaccessed" == "" ]
204            then
205               printprocesslog "INFO "$file" was accessed in the last 30 minutes => continue"
206               continue
207            else
208               printprocesslog "WARN: "$file" has empty TSTOP but was not touched for 30 minutes"
209               fileerror="yes"
210            fi
211         fi
212
213         # rsync
214         #if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile
215         if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
216         then
217            printprocesslog "ERROR something went wrong with rsync of "$file
218            rm $localfile
219            continue
220         fi
221         printprocesslog "INFO "$file" rsynced successfully."
222      fi
223
224      # for .drs.fits files no further treatment needed
225      if [ "`echo $localfile | grep -o drs`" == "drs" ]
226      then
227         continue
228      fi
229     
230#      # temporary check (needed to run on newdaq)
231#      if [ "`echo $file | grep -o drs`" == "drs" ]
232#      then
233#         nondrs=`basename $file | sed -e 's/[.]drs//g'`
234#         nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null `
235#         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
236#      else
237#         tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
238#      fi
239#      # when stop time is 0, file is not closed
240#      # when an error is returned the tstop is empty
241#      if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
242#      then
243#         printprocesslog "WARN "$file" not yet closed. -> continue (temporary check)"
244#         continue
245#      fi
246#      # end temporary check
247     
248      # treat other files (.fits.fz)
249      runtype=`$factpath/fitsdump -h $localfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z0-9._-]+[']" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
250      if [ "$runtype" != "data" ]
251      then
252         # skip a non-data run when it has not 1000 evts
253         #   as this means probably an fad-loss
254         #   and these runs are repeated in that case
255         numevts=`$factpath/fitsdump -h $file  2>/dev/null | grep Events | grep -E -o '[0-9]+'`
256         if [ "$numevts" == "" ]
257         then
258            printprocesslog "WARN could not get number of events from file "$file" -> continue "
259            #echo "WARN could not get number of events from file "$file" -> continue "
260            continue
261         fi
262         if [ $numevts -ne 1000 ]
263         then
264            printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")"
265            continue
266         fi
267      fi
268
269      # get run number
270      runnum=`basename $localfile | cut -d_ -f2 | cut -d. -f1`
271     
272      # what is needed to process the different runs?
273      #   P: run#(P), run#(drs-file)
274      #   C: run#(C), run#(drs-file), run#(drs-time)
275      #   D: run#(D), run#(drs-file), run#(drs-time), ?
276      # what is drs-file? pedestal, roi300, has drs.fits
277      callistolog=$calpath"/"$date"_"$runnum"-calibration.log"
278      case $runtype in
279         data) # treat D-runs
280            if [ "$fileerror" = "yes" ]
281            then
282               printprocesslog "INFO do not further process corrupted file "$localfile
283               fileerror=
284               continue
285            fi
286           
287            # some accounting
288            printprocesslog "DEBUG counting callisto logs and set data files +1."
289            # get number of callisto logs
290            runcallistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
291            # count runs to be calibrated
292            numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
293            printprocesslog "DEBUG running callistos: "$runcallistocount" #runs: "$numcalibrated" #callisto-logs: "${#callistofiles[@]}
294           
295            # do not overload system in case of a lot of files to be processed
296            #  numruncallistos is set in setup.fact.lp.data
297            if [ $runcallistocount -ge $numruncallistos ]
298            then
299               printprocesslog "INFO "$runcallistocount" RunCallisto.sh are running -> continue"
300               continue
301            fi
302           
303            # starting calibration
304            if ! [ -e $callistolog ]
305            then
306               rsync_aux_file $drivefile
307               rsync_aux_file $drivefile2
308               rsync_aux_file $mweatherfile
309               rsync_aux_file $ratesfile
310               rsync_aux_file $tempfile
311               rsync_aux_file  $humfile
312               
313               calfile=$calpath"/"$date"_"$runnum"_C.root"
314               printprocesslog "INFO starting RunCallisto.sh for drun "$localfile" logfile "$callistolog" drs-calib "$drscalib" outpath "$outpath" calfile "$calfile
315               `dirname $0`/RunCallisto.sh "drun" $callistolog $localfile $drscalib $calpath $calfile &
316            fi
317            continue
318            ;;
319         pedestal) # treat P-runs
320            roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
321            if [ $roi -eq 300 ]
322            then
323               # check drs-file
324               drsfile=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
325               if [ -e $drsfile ]
326               then
327                  # set name of drs-file
328                  drscalib=$drsfile
329                  continue
330               #else
331                  # not needed for QLA
332                  #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
333                  #if ! [ -e $callistolog ]
334                  #then
335                  #   pedfile=$calpath"/"$date"_"$runnum"-pedestal.root"
336                  #   # count runs to be calibrated
337                  #   printprocesslog "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
338                  #   echo "INFO starting RunCallisto.sh for prun "$localfile" logfile "$callistolog" drs-calib "$drscalib" pedfile "$pedfile
339                  #  `dirname $0`/RunCallisto.sh "prun" $callistolog $localfile $drscalib $pedfile &
340                  #fi
341               fi
342            fi
343            ;;
344         light-pulser-ext) # treat C-runs
345            # do lp-treatment -> not needed for QLA
346            #lpfile=$calpath"/"$date"_"$runnum"-lightpulser.root"
347            #numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
348            #if ! [ -e $callistolog ]
349            #then
350            #   if [ -e $drstime ]
351            #   then
352            #      # count runs to be calibrated
353            #      printprocesslog "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
354            #      echo "INFO starting RunCallisto.sh for crun "$localfile" logfile "$callistolog" drs-calib "$drscalib" drs-time "$drstime" lpfile "$lpfile
355            #      `dirname $0`/RunCallistoNew.sh "crun" $callistolog $localfile $drscalib $drstime $lpfile &
356            #   fi
357            #fi
358            ;;
359         drs-time) # treat drs-time runs
360            # do drs-timing calibration
361            drstime=$calpath"/"$date"_"$runnum"-drstime.root"
362            # starting calibration
363            numcalibrated=`echo " $numcalibrated + 1 " | bc -l`
364            if ! [ -e $callistolog ]
365            then
366               # count runs to be calibrated
367               printprocesslog "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
368               #echo "INFO starting RunCallisto.sh for time "$localfile" logfile "$callistolog" drs-ped "$drsped" drstime "$drstime
369               `dirname $0`/RunCallisto.sh "time" $callistolog $localfile $drsped $drstime &
370            fi
371            ;;
372         drs-pedestal) # get drs-pedestal
373            roi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep ROI | grep -v ROITM | grep -E -o "[0-9][0-9][0-9][0-9]?" | sed -e "s/'//g" -e "s/_/-/g" -e "s/[.]//g"`
374            drs=`$factpath/fitsdump -h $localfile  2>/dev/null | grep DRSCALIB | grep -E -o " T " `
375            if [ $roi -eq 1024 ] && [ "$drs" == " T " ]
376            then
377               drsped=`echo $localfile | sed -e 's/[.]fits[.]fz/.drs.fits/g'`
378            fi
379            ;;
380         *) # other runs
381            printprocesslog "INFO file "$file" has runtype "$runtype" -> continue "
382            continue
383            ;;
384      esac
385   done
386   printprocesslog "INFO status after loop: "$runcallistocount" callistos running, "$numcalibrated" data runs to process in total, "${#callistofiles[@]}" have already a callisto-logfile "
387   
388   # get new file lists
389   printprocesslog "INFO get new file lists for "$datepath
390   #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
391   files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
392   fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
393   callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
394   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
395   printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
396
397   # wait and get new file lists
398   update=
399   if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
400   then 
401      printprocesslog "INFO wait 60 seconds."
402      sleep 60
403      printprocesslog "INFO get new file lists for "$datepath
404      #files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
405      files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort ` )
406      fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
407      callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
408   fi
409   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
410   printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
411   sleep 30
412   printprocesslog "INFO sleep 30"
413done
414
415
416finish
Note: See TracBrowser for help on using the repository browser.