Changeset 12789 for trunk/DataCheck


Ignore:
Timestamp:
01/30/12 19:57:07 (13 years ago)
Author:
Daniela Dorner
Message:
included further checks and fill more information; common logfile
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/FillAuxData.sh

    r12691 r12789  
    11#!/bin/bash
    22
    3 today=`date +%F`
    4 logfile=/home/`whoami`/DataCheck/log/FillAuxData$today.log
    5 
    6 # steps:
    7 #  select run# from DB - no, date is given
    8 #  get daily files
    9 #  select file# from DB
    10 #  select start and stop from DB
    11 #  get info from files
    12 #  fill into into DB
    13 
    14 
    15 doupdate="yes" # update all entries
    16 doupdate="no" # fill only entries which are not yet existing
    17 
    18 echo ""  >> $logfile 2>&1
    19 echo ""  >> $logfile 2>&1
    20 echo ""  >> $logfile 2>&1
    21 echo `date`"executing "$0"..." >> $logfile 2>&1
    22 echo "=====> doupdate: "$doupdate >> $logfile 2>&1
    23 
    24 password=`cat /home/fact/DataCheck/.pw`
     3# missing:
     4#   vferify of files
     5#   only treat file if it is there
     6
     7# mjd of 1970-01-01
     8#   needed in this script as for 1 day the mjd in the aux files are inconsistent
     9mjdref=40587
     10
     11# option
     12doupdate="yes" # update all entries (needed when new fields have been added)
     13doupdate="no" # fill only entries which are not yet existing (default)
     14
     15source `dirname $0`/Sourcefile.sh
     16printprocesslog "INFO starting $0 with option doupdate="$doupdate
     17
     18# setup to use ftools
     19source $HEADAS/headas-init.sh
     20
     21pwfile=`dirname $0`/.pw
     22password=`cat $pwfile 2>/dev/null`
    2523if [ "$password" == "" ]
    2624then
    27    echo "please insert password in .pw file"
     25   echo "please insert password in $pwfile"
     26   printprocesslog "ERROR password for DB access in $pwfile missing"
     27   finish
    2828fi
    2929
    30 # setup to use ftools
    31 export HEADAS=/opt/heasoft-6.11/x86_64-unknown-linux-gnu-libc2.13-0/
    32 source $HEADAS/headas-init.sh
     30# check if software is available
     31if ! ls $factpath/fitsdump >/dev/null 2>&1
     32then
     33   printprocesslog "ERROR "$factpath"/fitsdump is not available."
     34   finish
     35fi
    3336
    3437# get last 2 nights
    3538dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
    36 #dates=( "2011/11/29" )
     39#dates=( `date +%Y/%m/%d --date="-12hour"` )
     40dates=( "2011/12/07" )
    3741# problems with 2011/11/29
    38 
    39 # do rsync for rawfiles of these dates
     42dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw
     43printprocesslog "INFO processing the following night: "${dates[@]}
     44
     45# do filling of aux data
    4046for date in ${dates[@]}
    4147do
    42    echo "" >> $logfile 2>&1
    43    echo "" >> $logfile 2>&1
    44    echo "" >> $logfile 2>&1
    45    auxdir=/loc_data/aux/$date
     48   auxdir=$auxdata/$date
    4649   runnumber=`echo $date | sed -e 's/\///g'`
    47    echo `date`": processing files in "$auxdir >> $logfile 2>&1
    4850   # check if data are available from that night
    4951   if ! [ -d $auxdir ]
    5052   then
    51       echo `date`": no data available in "$auxdir >> $logfile 2>&1
     53      printprocesslog "INFO no data available in "$auxdir
    5254      continue
     55   else
     56      printprocesslog "INFO processing files in "$auxdir
    5357   fi
    5458
    … …  
    5761   if ! [ -e $trackingfile ]
    5862   then
    59       echo $trackingfile" not found"
     63      printprocesslog "WARN "$trackingfile" not found."
    6064   fi
    6165   triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
    6266   if ! [ -e $triggerratefile ]
    6367   then
    64       echo $triggerratefile" not found"
     68      printprocesslog "WARN "$triggerratefile" not found."
    6569   fi
    6670   thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
    6771   if ! [ -e $thresholdfile ]
    6872   then
    69       echo $thresholdfile" not found"
     73      printprocesslog "WARN "$thresholdfile" not found."
    7074   fi
    7175   biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
    7276   if ! [ -e $biasvoltagefile ]
    7377   then
    74       echo $biasvoltagefile" not found"
     78      printprocesslog "WARN "$biasvoltagefile" not found."
    7579   fi
    7680   biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
    7781   if ! [ -e $biascurrentsfile ]
    7882   then
    79       echo $biascurrentsfile" not found"
    80    fi
     83      printprocesslog "WARN "$biascurrentsfile" not found."
     84   fi
     85
     86
    8187
    8288   # get file numbers from DB
    8389   query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber
    84    filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>> $logfile` )
     90   printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query1
     91   filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>/dev/null` )
    8592   for filenum in ${filenumbers[@]}
    8693   do
    87       #echo $filenum
    88       #query2="SELECT fRunStart from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
    89       #starttime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query2" 2>> $logfile`
    90       #query3="SELECT fRunStop from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
    91       #stoptime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query3" 2>> $logfile`
    92       #echo "start: "$starttime
    93       #echo "stop: "$stoptime
     94      printprocesslog "INFO processing file number "$filenum
     95     
     96      # get information from rawfile
    9497      rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits
    95       if ! [ -e $trackingfile ]
    96       then
    97          echo "ERROR: "$rawfile" not found."
     98      if ! [ -e $rawfile ]
     99      then
     100         printprocesslog "ERROR: "$rawfile" not found."
    98101         continue
    99       #else
    100       #   echo $rawfile
    101       fi
    102       checkfitsfile=`fverify $rawfile  2>> $logfile | grep '0 error(s)'`
     102      fi
     103      checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
    103104      if [ "$checkfitsfile" == "" ]
    104105      then
    105106         numfitserrors=1
    106          echo "WARN: "$rawfile" probably corrupted."
     107         printprocesslog "WARN: "$rawfile" probably corrupted."
    107108         continue
    108109      fi
    109       runtype=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
    110       #echo " runtype: "$runtype
    111       #/home/fact/FACT++/fitsdump -h -t Events $rawfile
    112       mjdref=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    113       tstarti=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    114       tstartf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
    115       tstopi=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    116       tstopf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
     110      runtype=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
     111      mjdrefraw=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
     112      tstarti=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
     113      tstartf=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
     114      tstopi=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
     115      tstopf=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
    117116      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ]
    118117      then
    119          echo "WARN: "$rawfile" has start or stop time = 0."
     118         printprocesslog "WARN: "$rawfile" has start or stop time = 0."
    120119         continue
    121120      fi
     121      tstart=`echo " $tstarti + $tstartf + $mjdrefraw " | bc -l`
     122      tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.00011574 " | bc -l`  # 10 sec
     123      #tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.000023148 " | bc -l` # 2 sec
     124      tstop=`echo " $tstopi + $tstopf + $mjdrefraw " | bc -l`
    122125      # calculate start and stop times
    123       tstart=`echo " $tstarti + $mjdref + $tstartf " | bc -l`
    124       tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.00011574 " | bc -l`  # 10 sec
    125       #tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.000023148 " | bc -l` # 2 sec
    126       tstop=`echo " $tstopi + $mjdref + $tstopf " | bc -l`
    127       #echo " tstart in mjd: "$tstart
    128       #echo " tstop in mjd: "$tstop
     126      # 2011/11/22 MJDREF in DRIVE empty, Time > 55000
     127      # 2011/11/23 MJDREF in DRIVE not empty, Time > 55000
     128      # 2011/11/24 MJDREF in DRIVE not empty, Time > 15000
     129      # raw files
     130      # 2011/11/21 no MJDREF
     131      # 2011/11/22 MJDREF
     132      if [ $runnumber -eq 20111123 ]
     133      then
     134         # add mjdref for days were aux files were inconsistent
     135         tstart=`echo " $tstart + $mjdref " | bc -l`
     136         tstart2=`echo " $tstart2 + $mjdref " | bc -l`
     137         tstop=`echo " $tstop + $mjdref " | bc -l`
     138      fi
     139
     140      # build query to update runinfo in DB
    129141      query0="UPDATE RunInfo SET "
    130142      queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
    131143     
    132144      # get information from tracking
    133       #echo $trackingfile
    134       ramin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
    135       ramax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
    136       ramean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
    137       decmin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
    138       decmax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
    139       decmean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
    140       #echo " ra: "$ramin" - "$ramean" - "$ramax
    141       if [ "$ramin" == "$ramax" ] && [ "$ramean" != "" ]
     145      if [ -e $trackingfile ]
     146      then
     147         ramin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     148         ramax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     149         ramean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     150         decmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     151         decmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     152         decmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     153         zdmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     154         zdmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     155         zdmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     156         azmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     157         azmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     158         azmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     159         # RA
     160         if [ "$ramin" == "$ramax" ] && [ "$ramean" != "0" ]
     161         then
     162            query0=$query0" fRightAscension="$ramean
     163         else
     164            query0=$query0" fRightAscension=NULL"
     165         fi
     166         # Declination
     167         if [ "$decmin" == "$decmax" ] && [ "$decmean" != "0" ]
     168         then
     169            query0=$query0", fDeclination="$decmean
     170         else
     171            query0=$query0", fDeclination=NULL"
     172         fi
     173         # Zd
     174         if [ "$zdmin" != "" ] && [ "$zdmean" != "0" ]
     175         then
     176            query0=$query0", fZenithDistanceMin="$zdmin
     177         else
     178            query0=$query0", fZenithDistanceMin=NULL"
     179         fi
     180         if [ "$zdmean" != "" ] && [ "$zdmean" != "0" ]
     181         then
     182            query0=$query0", fZenithDistanceMean="$zdmean
     183         else
     184            query0=$query0", fZenithDistanceMean=NULL"
     185         fi
     186         if [ "$zdmax" != "" ] && [ "$zdmean" != "0" ]
     187         then
     188            query0=$query0", fZenithDistanceMax="$zdmax
     189         else
     190            query0=$query0", fZenithDistanceMax=NULL"
     191         fi
     192         # Az
     193         if [ "$azmin" != "" ] && [ "$azmean" != "0" ]
     194         then
     195            query0=$query0", fAzimuthMin="$azmin
     196         else
     197            query0=$query0", fAzimuthMin=NULL"
     198         fi
     199         if [ "$azmean" != "" ] && [ "$azmean" != "0" ]
     200         then
     201            query0=$query0", fAzimuthMean="$azmean
     202         else
     203            query0=$query0", fAzimuthMean=NULL"
     204         fi
     205         if [ "$azmax" != "" ] && [ "$azmean" != "0" ]
     206         then
     207            query0=$query0", fAzimuthMax="$azmax
     208         else
     209            query0=$query0", fAzimuthMax=NULL"
     210         fi
     211      fi
     212   
     213      # get information from trigger
     214      if ! [ -e $triggerratefile ]
     215      then
     216         ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     217         ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     218         ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     219         ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     220         if [ "$ratemedian" != "0" ]
     221         then
     222            query0=$query0", fTriggerRateMedian="$ratemedian
     223         else
     224            query0=$query0", fTriggerRateMedian=NULL"
     225         fi
     226      fi
     227     
     228      # get information from thresholds
     229      if ! [ -e $thresholdfile ]
     230      then
     231         threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     232         threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     233         threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     234         threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     235         if [ "$threshmean" == "0" ]
     236         then
     237            threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     238            threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     239            threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     240            threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     241            # is this taking into account the 160 values in a row?
     242         fi
     243         query0=$query0", fThresholdMedian="$threshmedian
     244      fi
     245
     246#         # get information from bias: Uref
     247#         biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     248#         biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     249#         biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     250#         biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     251#         if [ "$biasvoltmean" == "0" ]
     252#         then
     253#            biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     254#            biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     255#            biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     256#            biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     257#            # is this taking into account the 160 values in a row?
     258#         fi
     259
     260      # get information from bias: U
     261      if ! [ -e $biasvoltagefile ]
     262      then
     263         biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     264         biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     265         biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     266         biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     267         if [ "$biasvoltmean" == "0" ]
     268         then
     269            biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     270            biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     271            biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     272            biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     273            # is this taking into account the 160 values in a row?
     274         fi
     275         query0=$query0", fBiasVoltageMedian="$biasvoltmedian
     276      fi
     277     
     278      query0=$query0" "$queryend
     279      printprocesslog "DEBUG update DB entry with QUERY: "$query0
     280      # send query to DB
     281      if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >/dev/null 2>&1
    142282      then
    143          query0=$query0" fRightAscension="$ramean
    144       else
    145          query0=$query0" fRightAscension=NULL"
    146       fi
    147       #echo " dec: "$decmin" - "$decmean" - "$decmax
    148       if [ "$decmin" == "$decmax" ] && [ "$decmean" != "" ]
    149       then
    150          query0=$query0", fDeclination="$decmean
    151       else
    152          query0=$query0", fDeclination=NULL"
    153       fi
    154 
    155       # get information from trigger
    156       #echo $triggerratefile
    157       #ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat -
    158       ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    159       ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    160       ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    161       ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    162       #echo " rate: "$ratemin" - "$ratemedian" - "$ratemean" - "$ratemax
    163       if [ "$ratemedian" != "0" ]
    164       then
    165          query0=$query0", fTriggerRateMedian="$ratemedian
    166       else
    167          query0=$query0", fTriggerRateMedian=NULL"
    168       fi
    169 
    170       # get information from trigger
    171       #echo $thresholdfile
    172       #echo "ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
    173       threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    174       threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    175       threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    176       threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    177       #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
    178       if [ "$threshmean" == "0" ]
    179       then
    180          #echo "ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
    181          threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    182          threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    183          threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    184          threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    185          #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
    186          # is this taking into account the 160 values in a row?
    187       fi
    188       query0=$query0", fThresholdMedian="$threshmedian
    189 
    190 #      # get information from bias: Uref
    191 #      #echo $biasvoltagefile
    192 #      #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
    193 #      biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    194 #      biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    195 #      biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    196 #      biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    197 #      echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
    198 #      if [ "$biasvoltmean" == "0" ]
    199 #      then
    200 #         #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
    201 #         biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    202 #         biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    203 #         biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    204 #         biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    205 #         echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
    206 #         # is this taking into account the 160 values in a row?
    207 #      fi
    208 
    209       # get information from bias: U
    210       #echo $biasvoltagefile
    211       #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
    212       biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    213       biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    214       biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    215       biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    216       #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
    217       if [ "$biasvoltmean" == "0" ]
    218       then
    219          #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
    220          biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    221          biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    222          biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    223          biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    224          #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
    225          # is this taking into account the 160 values in a row?
    226       fi
    227       query0=$query0", fBiasVoltageMedian="$biasvoltmedian
    228      
    229       query0=$query0" "$queryend
    230      
    231       if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >> $logfile 2>&1
    232       then
    233          echo "insert/update of "$runnumber" "$filenum" to mysql failed" >> $logfile 2>&1
    234          echo "Q0: "$query0" --- query failed."
     283         printprocesslog "ERROR insert/update of "$runnumber" "$filenum" to mysql failed."
    235284      fi
    236285   done
    237286done
    238287
    239 
     288finish
     289
     290
Note: See TracChangeset for help on using the changeset viewer.