Index: trunk/DataCheck/FillAuxData.sh
===================================================================
--- trunk/DataCheck/FillAuxData.sh	(revision 12689)
+++ trunk/DataCheck/FillAuxData.sh	(revision 12691)
@@ -33,6 +33,7 @@
 
 # get last 2 nights
-dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` )
-dates=( "2011/11/23" )
+dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` )
+#dates=( "2011/11/29" )
+# problems with 2011/11/29
 
 # do rsync for rawfiles of these dates
@@ -84,8 +85,5 @@
    for filenum in ${filenumbers[@]}
    do
-      filenum=42
-      filenum=53
-      filenum=56
-      echo $filenum
+      #echo $filenum
       #query2="SELECT fRunStart from RunInfo WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
       #starttime=`/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query2" 2>> $logfile`
@@ -95,19 +93,43 @@
       #echo "stop: "$stoptime 
       rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits
+      if ! [ -e $trackingfile ]
+      then 
+         echo "ERROR: "$rawfile" not found."
+         continue
+      #else
+      #   echo $rawfile
+      fi
+      checkfitsfile=`fverify $rawfile  2>> $logfile | grep '0 error(s)'`
+      if [ "$checkfitsfile" == "" ]
+      then
+         numfitserrors=1
+         echo "WARN: "$rawfile" probably corrupted."
+         continue
+      fi
+      runtype=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
+      #echo " runtype: "$runtype
       #/home/fact/FACT++/fitsdump -h -t Events $rawfile 
       mjdref=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
       tstarti=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
       tstartf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'`
+      tstopi=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
+      tstopf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
+      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ]
+      then 
+         echo "WARN: "$rawfile" has start or stop time = 0."
+         continue
+      fi
+      # calculate start and stop times
       tstart=`echo " $tstarti + $mjdref + $tstartf " | bc -l`
       tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.00011574 " | bc -l`  # 10 sec
       #tstart2=`echo " $tstarti + $mjdref + $tstartf - 0.000023148 " | bc -l` # 2 sec
-      tstopi=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
-      tstopf=`/home/fact/FACT++/fitsdump -h -t Events $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'`
       tstop=`echo " $tstopi + $mjdref + $tstopf " | bc -l`
-      echo "tstart in mjd: "$tstart
-      echo "tstop in mjd: "$tstop
+      #echo " tstart in mjd: "$tstart
+      #echo " tstop in mjd: "$tstop
+      query0="UPDATE RunInfo SET "
+      queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum
       
       # get information from tracking
-      echo $trackingfile
+      #echo $trackingfile
       ramin=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[0-9][.][0-9]+'`
       ramax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
@@ -116,9 +138,21 @@
       decmax=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[0-9][.][0-9]+'`
       decmean=`ftcopy $trackingfile'[Time> '${tstart}' && Time< '${tstop}'][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[0-9][.][0-9]+'`
-      echo "ra: "$ramin" - "$ramean" - "$ramax
-      echo "dec: "$decmin" - "$decmean" - "$decmax
+      #echo " ra: "$ramin" - "$ramean" - "$ramax
+      if [ "$ramin" == "$ramax" ] && [ "$ramean" != "" ]
+      then
+         query0=$query0" fRightAscension="$ramean
+      else
+         query0=$query0" fRightAscension=NULL"
+      fi
+      #echo " dec: "$decmin" - "$decmean" - "$decmax
+      if [ "$decmin" == "$decmax" ] && [ "$decmean" != "" ]
+      then
+         query0=$query0", fDeclination="$decmean
+      else
+         query0=$query0", fDeclination=NULL"
+      fi
 
       # get information from trigger
-      echo $triggerratefile
+      #echo $triggerratefile
       #ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat -
       ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
@@ -126,85 +160,79 @@
       ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
       ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      echo "rate: "$ratemin" - "$ratemedian" - "$ratemean" - "$ratemax
+      #echo " rate: "$ratemin" - "$ratemedian" - "$ratemean" - "$ratemax
+      if [ "$ratemedian" != "0" ]
+      then
+         query0=$query0", fTriggerRateMedian="$ratemedian
+      else
+         query0=$query0", fTriggerRateMedian=NULL"
+      fi
 
       # get information from trigger
-      echo $thresholdfile
-      echo "ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
+      #echo $thresholdfile
+      #echo "ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
       threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
       threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
       threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
       threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      echo "threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
+      #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
       if [ "$threshmean" == "0" ]
       then
-         echo "ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
+         #echo "ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat -"
          threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
          threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
          threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
          threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' && Time< '${tstop}'][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-         echo "threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
+         #echo " threshold: "$threshmin" - "$threshmedian" - "$threshmean" - "$threshmax
          # is this taking into account the 160 values in a row?
       fi
-
-      # get information from bias: Uref
-      echo $biasvoltagefile
-      echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
-      biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      echo "bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
-      if [ "$biasvoltmean" == "0" ]
-      then
-         echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
-         biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
-         biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
-         biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
-         biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-         echo "bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
-         # is this taking into account the 160 values in a row?
-      fi
+      query0=$query0", fThresholdMedian="$threshmedian
+
+#      # get information from bias: Uref
+#      #echo $biasvoltagefile
+#      #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
+#      biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#      biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#      biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#      biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#      echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
+#      if [ "$biasvoltmean" == "0" ]
+#      then
+#         #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat -"
+#         biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#         biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#         biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#         biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
+#         echo " bias: "$biasvoltrefmin" - "$biasvoltrefmedian" - "$biasvoltrefmean" - "$biasvoltrefmax
+#         # is this taking into account the 160 values in a row?
+#      fi
 
       # get information from bias: U
-      echo $biasvoltagefile
-      echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
+      #echo $biasvoltagefile
+      #echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
       biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
       biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
       biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
       biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-      echo "bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
+      #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
       if [ "$biasvoltmean" == "0" ]
       then
-         echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
+         #echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat -"
          biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'`
          biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'`
          biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'`
          biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'`
-         echo "bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
+         #echo " bias: "$biasvoltmin" - "$biasvoltmedian" - "$biasvoltmean" - "$biasvoltmax
          # is this taking into account the 160 values in a row?
       fi
-
-      # get information from bias: status
-      echo $biasvoltagefile
-      echo "ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat -"
-      biasstatmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'min' | grep -E -o '[0-9]'`
-      biasstatmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'max' | grep -E -o '[0-9]'`
-      biasstatmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]'`
-      biasstatmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'median' | grep -E -o '[0-9]'`
-      echo "bias: "$biasstatmin" - "$biasstatmedian" - "$biasstatmean" - "$biasstatmax
-      if [ "$biasstatmean" == "0" ]
-      then
-         echo "ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat -"
-         biasstatmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'min' | grep -E -o '[0-9]'`
-         biasstatmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'max' | grep -E -o '[0-9]'`
-         biasstatmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]'`
-         biasstatmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col QoS;Time]' - | ftcopy -'[col QoS]' - | ftstat - | grep 'median' | grep -E -o '[0-9]'`
-         echo "bias: "$biasstatmin" - "$biasstatmedian" - "$biasstatmean" - "$biasstatmax
-         # is this taking into account the 160 values in a row?
-      fi
-
-      exit
+      query0=$query0", fBiasVoltageMedian="$biasvoltmedian
+      
+      query0=$query0" "$queryend
+      
+      if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >> $logfile 2>&1
+      then
+         echo "insert/update of "$runnumber" "$filenum" to mysql failed" >> $logfile 2>&1
+         echo "Q0: "$query0" --- query failed."
+      fi
    done
-   
 done