Changeset 17587 for trunk/DataCheck/QuickLook

Timestamp:

03/02/14 00:20:58 (11 years ago)

Author:

Daniela Dorner

Message:

implemented usage of variable certaindate, implemented check for cases that data are deleted in wrong order, added regexp to search for file list

File:

• trunk/DataCheck/QuickLook/Step1.sh (modified) (14 diffs)

trunk/DataCheck/QuickLook/Step1.sh

@@ -5 +5 @@
 
 # get date (before 18h there is no new data to be processed)
-datepath=`date --date="-19HOUR" +%Y/%m/%d`
-date=`date --date="-19HOUR" +%Y%m%d`
-# for processing by hand
-#datepath="2013/08/11"
-#date="20130811"
+if [ "$certaindate" != "" ]
+then
+   checkstring=`echo $certaindate | grep -E -o '20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]'`
+   if [ "$checkstring" = "" ]
+   then
+      echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
+      finish
+   fi
+   datepath=$certaindate
+else
+   datepath=`date --date="-19HOUR" +%Y/%m/%d`
+fi
+date=`echo $datepath | sed -e 's/\///g'`
 printprocesslog "INFO processing "$datepath
-night=`echo $datepath | sed -e 's/\///g'`
 
 auxpathnewdaq=/newdaq/aux/$datepath
@@ -29 +36 @@
 # needed auxiliary files: 
 #   drive file with information about current source position
-drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
-drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
+drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
+drivefilenewdaq=$auxpathnewdaq/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
 #   drive file with information about tracking position
-drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
-drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
+drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
+drivefilenewdaq2=$auxpathnewdaq/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
 #   file with magic weather information 
-mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
-mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
+mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
+mweatherfilenewdaq=$auxpathnewdaq/${date}.MAGIC_WEATHER_DATA.fits
 #   file with trigger rates
-ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
-ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
+ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
+ratesfilenewdaq=$auxpathnewdaq/${date}.FTM_CONTROL_TRIGGER_RATES.fits
 #   file with trigger rates
-tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
-tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
+tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
+tempfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_TEMPERATURE.fits
 #   file with trigger rates
-humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
-humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits
+humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
+humfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_HUMIDITY.fits
 
 function rsync_aux_file()
@@ -84 +91 @@
 
 printprocesslog "INFO get lists of raw files on newdaq and daq"
-files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
+files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
+# to treat links use:
+#files=( `find -L $rawpathnewdaq -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
+
 if [ ${#files[@]} -eq 0 ]
 then
@@ -90 +100 @@
    finish
 fi
-fileslocal=( `find $rawpath -type f | sort` )
+fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
 callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
-numdataruns=0
+# get number of dataruns from DB
+query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
+numdataruns=`sendquery`
 
 # create raw directory on daq, if not yet there
@@ -100 +112 @@
 printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
 
-while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ] || [ ${#callistofiles[@]} -eq 0 ]
+while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ] # || [ $numdataruns -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
 do
    # only continue with script 
@@ -117 +129 @@
    do
       printprocesslog "processing "$file
+      #echo "processing "$file
       localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
 
@@ -127 +140 @@
          if [ "`echo $file | grep -o drs`" == "drs" ]
          then
-            nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
+            nondrs=`basename $file | sed -e 's/[.]drs//g'`
+            nondrsfile=`find $rawpath -name $nondrs.*z`
+            #echo "nondrs: "$nondrsfile
             tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
          else
             tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
          fi
+         #echo "tstop -"$tstop"-"
          # when stop time is 0, file is not closed
-         if [ "$tstop" == "0" ]
+         # when an error is returned the tstop is empty
+         if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
          then
             printprocesslog "WARN "$file" not yet closed."
@@ -168 +185 @@
       fi
 
-
       if [ "`echo $localfile | grep -o drs`" != "drs" ]
       then
@@ -190 +206 @@
                continue
             fi
-            seqfile=$seqpath/${night}_${runnum}.seq
+            seqfile=$seqpath/${date}_${runnum}.seq
             printprocesslog "INFO write data-seq "$seqfile
             echo "# written by automatic analysis in LP" >$seqfile
             echo "" >> $seqfile
-            echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
+            echo "Sequence: "`echo $date | cut -c3-8`$runnum >> $seqfile
             echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
             echo "" >> $seqfile
@@ -253 +269 @@
                continue
             fi
-            callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$night"_"$runnum"-calibration.log"
+            callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$date"_"$runnum"-calibration.log"
             if ! [ -e $callistolog ]
             then
@@ -274 +290 @@
             if [ $numevts -ne 1000 ]
             then
-               printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$nmevts")"
+               printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")"
                continue
             fi
@@ -293 +309 @@
             run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
             run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
-            seqfile=$seqpath/${night}_${run1}.drs.seq
+            seqfile=$seqpath/${date}_${run1}.drs.seq
             printprocesslog "INFO write drs-seq "$seqfile
             echo "# written by automatic analysis in LP" > $seqfile
             echo "" >> $seqfile
-            echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
+            echo "Sequence: "`echo $date | cut -c3-8`$run1 >> $seqfile
             echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
             echo "" >> $seqfile
@@ -322 +338 @@
    # get new file lists
    printprocesslog "INFO get new file lists for "$datepath
-   files=( `find $rawpathnewdaq -type f | sort` )
-   fileslocal=( `find $rawpath -type f | sort` )
+   files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
+   fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
    callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
    #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
@@ -336 +352 @@
       #echo "sleep 60..."
       printprocesslog "INFO get new file lists for "$datepath
-      files=( `find $rawpathnewdaq -type f | sort` )
-      fileslocal=( `find $rawpath -type f | sort` )
+      files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
+      fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
       callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
    fi
    #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
    printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+   
+   # this should not happen, but may in rare cases if data are deleted in the wrong order
+   if [ ${#files[@]} -lt ${#fileslocal[@]} ]
+   then
+      printprocesslog "ERROR numfiles on newdaq("${#files[@]}") < #files on daq ("${#fileslocal[@]}") - maybe deleting data in the wrong order?"
+      echo "ERROR numfiles on newdaq("${#files[@]}") < #files on daq ("${#fileslocal[@]}") - maybe deleting data in the wrong order?"
+      finish
+   fi
 done