Changeset 12935 for trunk/DataCheck


Ignore:
Timestamp:
02/23/12 16:54:46 (13 years ago)
Author:
Daniela Dorner
Message:
implemented function for getting statistics; usage of sendquery
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/FillAuxData.sh

    r12789 r12935  
    77# mjd of 1970-01-01
    88#   needed in this script as for 1 day the mjd in the aux files are inconsistent
    9 mjdref=40587
     9# mjdref=40587
     10
     11# known:
     12# 2011/11/22 MJDREF in DRIVE empty, Time > 55000
     13# 2011/11/23 MJDREF in DRIVE not empty, Time > 55000
     14# 2011/11/24 MJDREF in DRIVE not empty, Time > 15000
     15# raw files
     16# 2011/11/21 no MJDREF
     17# 2011/11/22 MJDREF
     18# further things: https://www.fact-project.org/logbook/showthread.php?tid=67
     19
     20# trigger rate has as first value -1, but with using the median it should be fine
     21
     22errlog=/home/fact/SW.automatic.processing/DataCheck/log/error.log
     23
     24function getfitsstatistics()
     25{
     26   # $1 filename
     27   # $2 colname
     28   # $3 tstart
     29   # $4 tstop
     30   good=
     31   min=
     32   mean=
     33   median=
     34   max=
     35   tmpfile=`dirname $0`/`basename $1`.tmp
     36   echo "ftcopy $1'[col Time]' - | ftstat - | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'" >> $errlog
     37   timefromfile=`ftcopy $1'[col Time]' - 2>>$errlog | ftstat - 2>>$errlog | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'`
     38   if [ $timefromfile -gt 30000 ]
     39   then
     40      echo "ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $errlog
     41      #ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -
     42      ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - 2>>$errlog | ftcopy -'[col '${2}']' - 2>>$errlog | ftstat - 2>>$errlog > $tmpfile
     43   else
     44      echo "ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $errlog
     45      #ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -
     46      ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - 2>>$errlog | ftcopy -'[col '${2}']' - 2>>$errlog | ftstat - 2>>$errlog > $tmpfile
     47   fi
     48   good=`cat $tmpfile | grep 'good' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
     49   min=`cat $tmpfile | grep 'min' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
     50   mean=`cat $tmpfile | grep 'mean' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
     51   median=`cat $tmpfile | grep 'median' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
     52   max=`cat $tmpfile | grep 'max' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'`
     53   if [ "$good" = "" ]
     54   then
     55      printprocesslog "WARN couldn't get statistics from file $1 for run $date $file"
     56      rm $tmpfile
     57      finish
     58   fi
     59   #echo "good: "$good
     60   #echo "min: "$min
     61   #echo "max: "$max
     62   #echo "mean: "$max
     63   #echo "median: "$max
     64   rm $tmpfile
     65}
    1066
    1167# option
    … …  
    3591fi
    3692
     93
    3794# get last 2 nights
    3895dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
    39 #dates=( `date +%Y/%m/%d --date="-12hour"` )
    40 dates=( "2011/12/07" )
    41 # problems with 2011/11/29
    42 dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw
    43 printprocesslog "INFO processing the following night: "${dates[@]}
     96#dates=( `find $ziprawdata -mindepth 3 -type d | sort | sed "s/\${ziprawdata_for_sed}//g" | sed -e 's/^\///'` ) #all available dates in /loc_data/zipraw
     97#echo ${dates[@]}
     98dates=( "2012/02/21" "2012/02/20" "2012/02/19" "2012/02/18" )
     99printprocesslog "INFO processing the following night(s): "${dates[@]}
    44100
    45101# do filling of aux data
    … …  
    48104   auxdir=$auxdata/$date
    49105   runnumber=`echo $date | sed -e 's/\///g'`
    50    # check if data are available from that night
     106
     107   # check if aux files are available from that night
    51108   if ! [ -d $auxdir ]
    52109   then
    … …  
    57114   fi
    58115
     116   # get file numbers from DB
     117   #   but only for not-corrupted files
     118   query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 "
     119   printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query
     120   filenumbers=( `sendquery $query` )
     121   if [ ${#filenumbers} -eq 0 ]
     122   then
     123      printprocesslog "INFO No files found in the DB for night "$date
     124      continue
     125   fi
     126
    59127   # get daily fits files
    60128   trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
    … …  
    62130   then
    63131      printprocesslog "WARN "$trackingfile" not found."
    64    fi
     132   else
     133      tracknumerrors=`fverify $trackingfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
     134      if [ $tracknumerrors -gt 0 ]
     135      then
     136         printprocesslog "WARN for $trackingfile fverify returned "$tracknumerrors" error(s)."
     137      fi
     138   fi
     139   
    65140   triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
    66141   if ! [ -e $triggerratefile ]
    67142   then
    68143      printprocesslog "WARN "$triggerratefile" not found."
    69    fi
     144   else
     145      trignumerrors=`fverify $triggerratefile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
     146      if [ $trignumerrors -gt 0 ]
     147      then
     148         printprocesslog "WARN for $triggerratefile fverify returned "$trignumerrors" error(s)."
     149      fi
     150   fi
     151   
    70152   thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
    71153   if ! [ -e $thresholdfile ]
    72154   then
    73155      printprocesslog "WARN "$thresholdfile" not found."
    74    fi
     156   else
     157      treshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
     158      if [ $treshnumerrors -gt 0 ]
     159      then
     160         printprocesslog "WARN for $thresholdfile fverify returned "$treshnumerrors" error(s)."
     161      fi
     162   fi
     163   
    75164   biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
    76165   if ! [ -e $biasvoltagefile ]
    77166   then
    78167      printprocesslog "WARN "$biasvoltagefile" not found."
    79    fi
    80    biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
    81    if ! [ -e $biascurrentsfile ]
    82    then
    83       printprocesslog "WARN "$biascurrentsfile" not found."
    84    fi
    85 
    86 
    87 
    88    # get file numbers from DB
    89    query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber
    90    printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query1
    91    filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>/dev/null` )
     168   else
     169      biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
     170      if [ $biasnumerrors -gt 0 ]
     171      then
     172         printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)."
     173      fi
     174   fi
     175
     176   # fill auxiliary information for files
    92177   for filenum in ${filenumbers[@]}
    93178   do
    94       printprocesslog "INFO processing file number "$filenum
    95      
     179      printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum`
    96180      # get information from rawfile
    97       rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits
     181      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.gz
    98182      if ! [ -e $rawfile ]
    99183      then
    … …  
    101185         continue
    102186      fi
    103       checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
    104       if [ "$checkfitsfile" == "" ]
    105       then
    106          numfitserrors=1
    107          printprocesslog "WARN: "$rawfile" probably corrupted."
    108          continue
    109       fi
     187      #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
     188      #if [ "$checkfitsfile" == "" ]
     189      #then
     190      #   numfitserrors=1
     191      #   printprocesslog "WARN: "$rawfile" probably corrupted."
     192      #   continue
     193      #fi
    110194      runtype=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
    111195      mjdrefraw=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    … …  
    114198      tstopi=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    115199      tstopf=`$factpath/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
    116       if [ "$tstarti" == "" ] || [ "$tstopi" == "" ]
    117       then
    118          printprocesslog "WARN: "$rawfile" has start or stop time = 0."
     200      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
     201      then
     202         printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
    119203         continue
    120204      fi
    121       tstart=`echo " $tstarti + $tstartf + $mjdrefraw " | bc -l`
    122       tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.00011574 " | bc -l`  # 10 sec
    123       #tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.000023148 " | bc -l` # 2 sec
    124       tstop=`echo " $tstopi + $tstopf + $mjdrefraw " | bc -l`
    125       # calculate start and stop times
    126       # 2011/11/22 MJDREF in DRIVE empty, Time > 55000
    127       # 2011/11/23 MJDREF in DRIVE not empty, Time > 55000
    128       # 2011/11/24 MJDREF in DRIVE not empty, Time > 15000
    129       # raw files
    130       # 2011/11/21 no MJDREF
    131       # 2011/11/22 MJDREF
    132       if [ $runnumber -eq 20111123 ]
    133       then
    134          # add mjdref for days were aux files were inconsistent
    135          tstart=`echo " $tstart + $mjdref " | bc -l`
    136          tstart2=`echo " $tstart2 + $mjdref " | bc -l`
    137          tstop=`echo " $tstop + $mjdref " | bc -l`
    138       fi
     205      # assuming that at least TSTARTI and TSTOPI are consistent
     206      #echo $rawfile
     207      #echo $tstarti
     208      #echo $tstopi
     209      #echo $tstartf
     210      #echo $tstopf
     211      if [ $tstarti -gt 30000 ]
     212      then
     213         tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
     214         tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
     215         #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
     216         tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
     217      else
     218         tstart=`echo " $tstarti + $tstartf " | bc -l`
     219         tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
     220         #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
     221         tstop=`echo " $tstopi + $tstopf " | bc -l`
     222      fi
     223      #echo $tstart
     224      #echo $tstop
     225      #if [ $runnumber -eq 20111123 ]
     226      #then
     227      #   # add mjdref for days were aux files were inconsistent
     228      #   tstart=`echo " $tstart + $mjdref " | bc -l`
     229      #   tstart2=`echo " $tstart2 + $mjdref " | bc -l`
     230      #   tstop=`echo " $tstop + $mjdref " | bc -l`
     231      #fi
    139232
    140233      # build query to update runinfo in DB
    141       query0="UPDATE RunInfo SET "
    142       queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
     234      query="UPDATE RunInfo SET "
    143235     
    144236      # get information from tracking
    145       if [ -e $trackingfile ]
    146       then
    147          ramin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    148          ramax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    149          ramean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    150          decmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    151          decmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    152          decmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    153          zdmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    154          zdmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    155          zdmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    156          azmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    157          azmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
    158          azmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'`
     237      if [ -e $trackingfile ] && [ $tracknumerrors -eq 0 ]
     238      then
    159239         # RA
    160          if [ "$ramin" == "$ramax" ] && [ "$ramean" != "0" ]
    161          then
    162             query0=$query0" fRightAscension="$ramean
    163          else
    164             query0=$query0" fRightAscension=NULL"
     240         getfitsstatistics $trackingfile "Ra" $tstart $tstop
     241         if [ "$min" == "$max" ] && [ $good -gt 0 ]
     242         then
     243            query=$query" fRightAscension="$mean
     244         else
     245            query=$query" fRightAscension=NULL"
     246            if [ $good -gt 0 ]
     247            then
     248               printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")."
     249            fi
    165250         fi
    166251         # Declination
    167          if [ "$decmin" == "$decmax" ] && [ "$decmean" != "0" ]
    168          then
    169             query0=$query0", fDeclination="$decmean
    170          else
    171             query0=$query0", fDeclination=NULL"
     252         getfitsstatistics $trackingfile "Dec" $tstart $tstop
     253         if [ "$decmin" == "$decmax" ] && [ $good -gt 0 ]
     254         then
     255            query=$query", fDeclination="$mean
     256         else
     257            query=$query", fDeclination=NULL"
     258            if [ $good -gt 0 ]
     259            then
     260               printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")."
     261            fi
    172262         fi
    173263         # Zd
    174          if [ "$zdmin" != "" ] && [ "$zdmean" != "0" ]
    175          then
    176             query0=$query0", fZenithDistanceMin="$zdmin
    177          else
    178             query0=$query0", fZenithDistanceMin=NULL"
    179          fi
    180          if [ "$zdmean" != "" ] && [ "$zdmean" != "0" ]
    181          then
    182             query0=$query0", fZenithDistanceMean="$zdmean
    183          else
    184             query0=$query0", fZenithDistanceMean=NULL"
    185          fi
    186          if [ "$zdmax" != "" ] && [ "$zdmean" != "0" ]
    187          then
    188             query0=$query0", fZenithDistanceMax="$zdmax
    189          else
    190             query0=$query0", fZenithDistanceMax=NULL"
     264         getfitsstatistics $trackingfile "Zd" $tstart $tstop
     265         if [ $good -gt 0 ]
     266         then
     267            query=$query", fZenithDistanceMin="$min
     268            query=$query", fZenithDistanceMean="$mean
     269            query=$query", fZenithDistanceMax="$max
     270         else
     271            query=$query", fZenithDistanceMin=NULL"
     272            query=$query", fZenithDistanceMean=NULL"
     273            query=$query", fZenithDistanceMax=NULL"
    191274         fi
    192275         # Az
    193          if [ "$azmin" != "" ] && [ "$azmean" != "0" ]
    194          then
    195             query0=$query0", fAzimuthMin="$azmin
    196          else
    197             query0=$query0", fAzimuthMin=NULL"
    198          fi
    199          if [ "$azmean" != "" ] && [ "$azmean" != "0" ]
    200          then
    201             query0=$query0", fAzimuthMean="$azmean
    202          else
    203             query0=$query0", fAzimuthMean=NULL"
    204          fi
    205          if [ "$azmax" != "" ] && [ "$azmean" != "0" ]
    206          then
    207             query0=$query0", fAzimuthMax="$azmax
    208          else
    209             query0=$query0", fAzimuthMax=NULL"
    210          fi
     276         getfitsstatistics $trackingfile "Az" $tstart $tstop
     277         if [ $good -gt 0 ]
     278         then
     279            query=$query", fAzimuthMin="$min
     280            query=$query", fAzimuthMean="$mean
     281            query=$query", fAzimuthMax="$max
     282         else
     283            query=$query", fAzimuthMin=NULL"
     284            query=$query", fAzimuthMean=NULL"
     285            query=$query", fAzimuthMax=NULL"
     286         fi
     287      else
     288         query=$query" fRightAscension=NULL"
     289         query=$query", fDeclination=NULL"
     290         query=$query", fZenithDistanceMin=NULL"
     291         query=$query", fZenithDistanceMean=NULL"
     292         query=$query", fZenithDistanceMax=NULL"
     293         query=$query", fAzimuthMin=NULL"
     294         query=$query", fAzimuthMean=NULL"
     295         query=$query", fAzimuthMax=NULL"
    211296      fi
    212297   
    213298      # get information from trigger
    214       if ! [ -e $triggerratefile ]
    215       then
    216          ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    217          ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    218          ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    219          ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    220          if [ "$ratemedian" != "0" ]
    221          then
    222             query0=$query0", fTriggerRateMedian="$ratemedian
    223          else
    224             query0=$query0", fTriggerRateMedian=NULL"
    225          fi
     299      if [ -e $triggerratefile ] && [ $trignumerrors -eq 0 ]
     300      then
     301         #echo " $triggerratefile TriggerRate $tstart $tstop"
     302         getfitsstatistics $triggerratefile "TriggerRate" $tstart $tstop
     303#         if [ "$mjdreftrig" == "" ]
     304#         then
     305#            ratemin=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     306#            ratemax=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     307#            ratemean=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     308#            ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     309#         else
     310#            ratemin=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
     311#            ratemax=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
     312#            ratemean=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
     313#            ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}'  && Time< '${tstop}'  && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
     314#         fi
     315         if [ $good -gt 0 ]
     316         then
     317            query=$query", fTriggerRateMedian="$median
     318         else
     319            query=$query", fTriggerRateMedian=NULL"
     320         fi
     321      else
     322         query=$query", fTriggerRateMedian=NULL"
    226323      fi
    227324     
    228325      # get information from thresholds
    229       if ! [ -e $thresholdfile ]
    230       then
    231          threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    232          threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    233          threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    234          threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    235          if [ "$threshmean" == "0" ]
    236          then
    237             threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    238             threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    239             threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    240             threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    241             # is this taking into account the 160 values in a row?
    242          fi
    243          query0=$query0", fThresholdMedian="$threshmedian
    244       fi
    245 
    246 #         # get information from bias: Uref
    247 #         biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    248 #         biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    249 #         biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    250 #         biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    251 #         if [ "$biasvoltmean" == "0" ]
    252 #         then
    253 #            biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    254 #            biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    255 #            biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    256 #            biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    257 #            # is this taking into account the 160 values in a row?
    258 #         fi
     326      if [ -e $thresholdfile ] && [ $treshnumerrors -eq 0 ]
     327      then
     328         getfitsstatistics $thresholdfile "PatchThresh" $tstart $tstop
     329         if [ $good -eq 0 ]
     330         then
     331            getfitsstatistics $thresholdfile "PatchThresh" $tstart2 $tstop
     332         fi
     333         if [ $good -gt 0 ]
     334         then
     335            query=$query", fThresholdMedian="$median
     336         else
     337            query=$query", fThresholdMedian=NULL"
     338         fi
     339      else
     340         query=$query", fThresholdMedian=NULL"
     341      fi
    259342
    260343      # get information from bias: U
    261       if ! [ -e $biasvoltagefile ]
    262       then
    263          biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    264          biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    265          biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    266          biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    267          if [ "$biasvoltmean" == "0" ]
    268          then
    269             biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
    270             biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
    271             biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
    272             biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
    273             # is this taking into account the 160 values in a row?
    274          fi
    275          query0=$query0", fBiasVoltageMedian="$biasvoltmedian
     344      if [ -e $biasvoltagefile ] && [ $biasnumerrors -eq 0 ]
     345      then
     346         getfitsstatistics $biasvoltagefile "U" $tstart $tstop
     347         if [ $good -eq 0 ]
     348         then
     349            getfitsstatistics $biasvoltagefile "U" $tstart2 $tstop
     350         fi
     351         if [ $good -gt 0 ]
     352         then
     353            query=$query", fBiasVoltageMedian="$median
     354         else
     355            query=$query", fBiasVoltageMedian=NULL"
     356         fi
     357      else
     358         query=$query", fBiasVoltageMedian=NULL"
    276359      fi
    277360     
    278       query0=$query0" "$queryend
    279       printprocesslog "DEBUG update DB entry with QUERY: "$query0
     361      # add where condition
     362      query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum
     363     
    280364      # send query to DB
    281       if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >/dev/null 2>&1
    282       then
    283          printprocesslog "ERROR insert/update of "$runnumber" "$filenum" to mysql failed."
    284       fi
     365      sendquery >/dev/null
    285366   done
    286367done
Note: See TracChangeset for help on using the changeset viewer.