Changeset 17587 for trunk/DataCheck


Ignore:
Timestamp:
03/02/14 00:20:58 (11 years ago)
Author:
Daniela Dorner
Message:
implemented usage of variable certaindate, implemented check for cases that data are deleted in wrong order, added regexp to search for file list
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/QuickLook/Step1.sh

    r17153 r17587  
    55
    66# get date (before 18h there is no new data to be processed)
    7 datepath=`date --date="-19HOUR" +%Y/%m/%d`
    8 date=`date --date="-19HOUR" +%Y%m%d`
    9 # for processing by hand
    10 #datepath="2013/08/11"
    11 #date="20130811"
     7if [ "$certaindate" != "" ]
     8then
     9   checkstring=`echo $certaindate | grep -E -o '20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]'`
     10   if [ "$checkstring" = "" ]
     11   then
     12      echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
     13      finish
     14   fi
     15   datepath=$certaindate
     16else
     17   datepath=`date --date="-19HOUR" +%Y/%m/%d`
     18fi
     19date=`echo $datepath | sed -e 's/\///g'`
    1220printprocesslog "INFO processing "$datepath
    13 night=`echo $datepath | sed -e 's/\///g'`
    1421
    1522auxpathnewdaq=/newdaq/aux/$datepath
    … …  
    2936# needed auxiliary files:
    3037#   drive file with information about current source position
    31 drivefile=$auxpath/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
    32 drivefilenewdaq=$auxpathnewdaq/${night}.DRIVE_CONTROL_SOURCE_POSITION.fits
     38drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
     39drivefilenewdaq=$auxpathnewdaq/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
    3340#   drive file with information about tracking position
    34 drivefile2=$auxpath/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
    35 drivefilenewdaq2=$auxpathnewdaq/${night}.DRIVE_CONTROL_TRACKING_POSITION.fits
     41drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
     42drivefilenewdaq2=$auxpathnewdaq/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
    3643#   file with magic weather information
    37 mweatherfile=$auxpath/${night}.MAGIC_WEATHER_DATA.fits
    38 mweatherfilenewdaq=$auxpathnewdaq/${night}.MAGIC_WEATHER_DATA.fits
     44mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
     45mweatherfilenewdaq=$auxpathnewdaq/${date}.MAGIC_WEATHER_DATA.fits
    3946#   file with trigger rates
    40 ratesfile=$auxpath/${night}.FTM_CONTROL_TRIGGER_RATES.fits
    41 ratesfilenewdaq=$auxpathnewdaq/${night}.FTM_CONTROL_TRIGGER_RATES.fits
     47ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
     48ratesfilenewdaq=$auxpathnewdaq/${date}.FTM_CONTROL_TRIGGER_RATES.fits
    4249#   file with trigger rates
    43 tempfile=$auxpath/${night}.FSC_CONTROL_TEMPERATURE.fits
    44 tempfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_TEMPERATURE.fits
     50tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
     51tempfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_TEMPERATURE.fits
    4552#   file with trigger rates
    46 humfile=$auxpath/${night}.FSC_CONTROL_HUMIDITY.fits
    47 humfilenewdaq=$auxpathnewdaq/${night}.FSC_CONTROL_HUMIDITY.fits
     53humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
     54humfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_HUMIDITY.fits
    4855
    4956function rsync_aux_file()
    … …  
    8491
    8592printprocesslog "INFO get lists of raw files on newdaq and daq"
    86 files=( `find $rawpathnewdaq -type f 2>/dev/null | sort` )
     93files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
     94# to treat links use:
     95#files=( `find -L $rawpathnewdaq -regex '.*[.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
     96
    8797if [ ${#files[@]} -eq 0 ]
    8898then
    … …  
    90100   finish
    91101fi
    92 fileslocal=( `find $rawpath -type f | sort` )
     102fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
    93103callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
    94 numdataruns=0
     104# get number of dataruns from DB
     105query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
     106numdataruns=`sendquery`
    95107
    96108# create raw directory on daq, if not yet there
    … …  
    100112printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
    101113
    102 while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ] || [ ${#callistofiles[@]} -eq 0 ]
     114while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ] # || [ $numdataruns -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
    103115do
    104116   # only continue with script
    … …  
    117129   do
    118130      printprocesslog "processing "$file
     131      #echo "processing "$file
    119132      localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
    120133
    … …  
    127140         if [ "`echo $file | grep -o drs`" == "drs" ]
    128141         then
    129             nondrsfile=`echo $file | sed -e 's/[.]drs//g'`
     142            nondrs=`basename $file | sed -e 's/[.]drs//g'`
     143            nondrsfile=`find $rawpath -name $nondrs.*z`
     144            #echo "nondrs: "$nondrsfile
    130145            tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
    131146         else
    132147            tstop=`$factpath/fitsdump -h $file  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
    133148         fi
     149         #echo "tstop -"$tstop"-"
    134150         # when stop time is 0, file is not closed
    135          if [ "$tstop" == "0" ]
     151         # when an error is returned the tstop is empty
     152         if [ "$tstop" == "0" ] || [ "$tstop" == "" ]
    136153         then
    137154            printprocesslog "WARN "$file" not yet closed."
    … …  
    168185      fi
    169186
    170 
    171187      if [ "`echo $localfile | grep -o drs`" != "drs" ]
    172188      then
    … …  
    190206               continue
    191207            fi
    192             seqfile=$seqpath/${night}_${runnum}.seq
     208            seqfile=$seqpath/${date}_${runnum}.seq
    193209            printprocesslog "INFO write data-seq "$seqfile
    194210            echo "# written by automatic analysis in LP" >$seqfile
    195211            echo "" >> $seqfile
    196             echo "Sequence: "`echo $night | cut -c3-8`$runnum >> $seqfile
     212            echo "Sequence: "`echo $date | cut -c3-8`$runnum >> $seqfile
    197213            echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
    198214            echo "" >> $seqfile
    … …  
    253269               continue
    254270            fi
    255             callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$night"_"$runnum"-calibration.log"
     271            callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$date"_"$runnum"-calibration.log"
    256272            if ! [ -e $callistolog ]
    257273            then
    … …  
    274290            if [ $numevts -ne 1000 ]
    275291            then
    276                printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$nmevts")"
     292               printprocesslog "INFO file "$file" is a non-data file ("$runtype") and has not 1000 events ("$numevts")"
    277293               continue
    278294            fi
    … …  
    293309            run6=`echo $runrow2 | cut -d_ -f6 | sed -e 's/pedestal300//g'`
    294310            run7=`echo $runrow2 | cut -d_ -f7 | sed -e 's/pedestal300//g'`
    295             seqfile=$seqpath/${night}_${run1}.drs.seq
     311            seqfile=$seqpath/${date}_${run1}.drs.seq
    296312            printprocesslog "INFO write drs-seq "$seqfile
    297313            echo "# written by automatic analysis in LP" > $seqfile
    298314            echo "" >> $seqfile
    299             echo "Sequence: "`echo $night | cut -c3-8`$run1 >> $seqfile
     315            echo "Sequence: "`echo $date | cut -c3-8`$run1 >> $seqfile
    300316            echo "Night: "`echo $datepath | sed -e 's/\//-/g'` >> $seqfile
    301317            echo "" >> $seqfile
    … …  
    322338   # get new file lists
    323339   printprocesslog "INFO get new file lists for "$datepath
    324    files=( `find $rawpathnewdaq -type f | sort` )
    325    fileslocal=( `find $rawpath -type f | sort` )
     340   files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
     341   fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
    326342   callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
    327343   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
    … …  
    336352      #echo "sleep 60..."
    337353      printprocesslog "INFO get new file lists for "$datepath
    338       files=( `find $rawpathnewdaq -type f | sort` )
    339       fileslocal=( `find $rawpath -type f | sort` )
     354      files=( `find $rawpathnewdaq -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
     355      fileslocal=( `find $rawpath -type f -regex '.*[.]fits[.]?[g]?[f]?[z]?' | sort` )
    340356      callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
    341357   fi
    342358   #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
    343359   printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
     360   
     361   # this should not happen, but may in rare cases if data are deleted in the wrong order
     362   if [ ${#files[@]} -lt ${#fileslocal[@]} ]
     363   then
     364      printprocesslog "ERROR numfiles on newdaq("${#files[@]}") < #files on daq ("${#fileslocal[@]}") - maybe deleting data in the wrong order?"
     365      echo "ERROR numfiles on newdaq("${#files[@]}") < #files on daq ("${#fileslocal[@]}") - maybe deleting data in the wrong order?"
     366      finish
     367   fi
    344368done
    345369
Note: See TracChangeset for help on using the changeset viewer.