Index: trunk/DataCheck/FillAuxData.sh =================================================================== --- trunk/DataCheck/FillAuxData.sh (revision 12934) +++ trunk/DataCheck/FillAuxData.sh (revision 12935) @@ -7,5 +7,61 @@ # mjd of 1970-01-01 # needed in this script as for 1 day the mjd in the aux files are inconsistent -mjdref=40587 +# mjdref=40587 + +# known: +# 2011/11/22 MJDREF in DRIVE empty, Time > 55000 +# 2011/11/23 MJDREF in DRIVE not empty, Time > 55000 +# 2011/11/24 MJDREF in DRIVE not empty, Time > 15000 +# raw files +# 2011/11/21 no MJDREF +# 2011/11/22 MJDREF +# further things: https://www.fact-project.org/logbook/showthread.php?tid=67 + +# trigger rate has as first value -1, but with using the median it should be fine + +errlog=/home/fact/SW.automatic.processing/DataCheck/log/error.log + +function getfitsstatistics() +{ + # $1 filename + # $2 colname + # $3 tstart + # $4 tstop + good= + min= + mean= + median= + max= + tmpfile=`dirname $0`/`basename $1`.tmp + echo "ftcopy $1'[col Time]' - | ftstat - | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'" >> $errlog + timefromfile=`ftcopy $1'[col Time]' - 2>>$errlog | ftstat - 2>>$errlog | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'` + if [ $timefromfile -gt 30000 ] + then + echo "ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $errlog + #ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - + ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - 2>>$errlog | ftcopy -'[col '${2}']' - 2>>$errlog | ftstat - 2>>$errlog > $tmpfile + else + echo "ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $errlog + #ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - + ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - 2>>$errlog | ftcopy -'[col '${2}']' - 2>>$errlog | ftstat - 2>>$errlog > $tmpfile + fi + good=`cat $tmpfile | grep 'good' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + min=`cat $tmpfile | grep 'min' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + mean=`cat $tmpfile | grep 'mean' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + median=`cat $tmpfile | grep 'median' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + max=`cat $tmpfile | grep 'max' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + if [ "$good" = "" ] + then + printprocesslog "WARN couldn't get statistics from file $1 for run $date $file" + rm $tmpfile + finish + fi + #echo "good: "$good + #echo "min: "$min + #echo "max: "$max + #echo "mean: "$max + #echo "median: "$max + rm $tmpfile +} # option @@ -35,11 +91,11 @@ fi + # get last 2 nights dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` ) -#dates=( `date +%Y/%m/%d --date="-12hour"` ) -dates=( "2011/12/07" ) -# problems with 2011/11/29 -dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw -printprocesslog "INFO processing the following night: "${dates[@]} +#dates=( `find $ziprawdata -mindepth 3 -type d | sort | sed "s/\${ziprawdata_for_sed}//g" | sed -e 's/^\///'` ) #all available dates in /loc_data/zipraw +#echo ${dates[@]} +dates=( "2012/02/21" "2012/02/20" "2012/02/19" "2012/02/18" ) +printprocesslog "INFO processing the following night(s): "${dates[@]} # do filling of aux data @@ -48,5 +104,6 @@ auxdir=$auxdata/$date runnumber=`echo $date | sed -e 's/\///g'` - # check if data are available from that night + + # check if aux files are available from that night if ! [ -d $auxdir ] then @@ -57,4 +114,15 @@ fi + # get file numbers from DB + # but only for not-corrupted files + query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 " + printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query + filenumbers=( `sendquery $query` ) + if [ ${#filenumbers} -eq 0 ] + then + printprocesslog "INFO No files found in the DB for night "$date + continue + fi + # get daily fits files trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits @@ -62,38 +130,54 @@ then printprocesslog "WARN "$trackingfile" not found." - fi + else + tracknumerrors=`fverify $trackingfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $tracknumerrors -gt 0 ] + then + printprocesslog "WARN for $trackingfile fverify returned "$tracknumerrors" error(s)." + fi + fi + triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits if ! [ -e $triggerratefile ] then printprocesslog "WARN "$triggerratefile" not found." - fi + else + trignumerrors=`fverify $triggerratefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $trignumerrors -gt 0 ] + then + printprocesslog "WARN for $triggerratefile fverify returned "$trignumerrors" error(s)." + fi + fi + thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits if ! [ -e $thresholdfile ] then printprocesslog "WARN "$thresholdfile" not found." - fi + else + treshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $treshnumerrors -gt 0 ] + then + printprocesslog "WARN for $thresholdfile fverify returned "$treshnumerrors" error(s)." + fi + fi + biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits if ! [ -e $biasvoltagefile ] then printprocesslog "WARN "$biasvoltagefile" not found." - fi - biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits - if ! [ -e $biascurrentsfile ] - then - printprocesslog "WARN "$biascurrentsfile" not found." - fi - - - - # get file numbers from DB - query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber - printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query1 - filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>/dev/null` ) + else + biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $biasnumerrors -gt 0 ] + then + printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)." + fi + fi + + # fill auxiliary information for files for filenum in ${filenumbers[@]} do - printprocesslog "INFO processing file number "$filenum - + printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum` # get information from rawfile - rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits + rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.gz if ! [ -e $rawfile ] then @@ -101,11 +185,11 @@ continue fi - checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` - if [ "$checkfitsfile" == "" ] - then - numfitserrors=1 - printprocesslog "WARN: "$rawfile" probably corrupted." - continue - fi + #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` + #if [ "$checkfitsfile" == "" ] + #then + # numfitserrors=1 + # printprocesslog "WARN: "$rawfile" probably corrupted." + # continue + #fi runtype=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` mjdrefraw=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` @@ -114,173 +198,170 @@ tstopi=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` tstopf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` - if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] - then - printprocesslog "WARN: "$rawfile" has start or stop time = 0." + if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ] + then + printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF " continue fi - tstart=`echo " $tstarti + $tstartf + $mjdrefraw " | bc -l` - tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.00011574 " | bc -l` # 10 sec - #tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.000023148 " | bc -l` # 2 sec - tstop=`echo " $tstopi + $tstopf + $mjdrefraw " | bc -l` - # calculate start and stop times - # 2011/11/22 MJDREF in DRIVE empty, Time > 55000 - # 2011/11/23 MJDREF in DRIVE not empty, Time > 55000 - # 2011/11/24 MJDREF in DRIVE not empty, Time > 15000 - # raw files - # 2011/11/21 no MJDREF - # 2011/11/22 MJDREF - if [ $runnumber -eq 20111123 ] - then - # add mjdref for days were aux files were inconsistent - tstart=`echo " $tstart + $mjdref " | bc -l` - tstart2=`echo " $tstart2 + $mjdref " | bc -l` - tstop=`echo " $tstop + $mjdref " | bc -l` - fi + # assuming that at least TSTARTI and TSTOPI are consistent + #echo $rawfile + #echo $tstarti + #echo $tstopi + #echo $tstartf + #echo $tstopf + if [ $tstarti -gt 30000 ] + then + tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l` + tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec + #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec + tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l` + else + tstart=`echo " $tstarti + $tstartf " | bc -l` + tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec + #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec + tstop=`echo " $tstopi + $tstopf " | bc -l` + fi + #echo $tstart + #echo $tstop + #if [ $runnumber -eq 20111123 ] + #then + # # add mjdref for days were aux files were inconsistent + # tstart=`echo " $tstart + $mjdref " | bc -l` + # tstart2=`echo " $tstart2 + $mjdref " | bc -l` + # tstop=`echo " $tstop + $mjdref " | bc -l` + #fi # build query to update runinfo in DB - query0="UPDATE RunInfo SET " - queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum + query="UPDATE RunInfo SET " # get information from tracking - if [ -e $trackingfile ] - then - ramin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - ramax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - ramean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - decmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - decmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - decmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - zdmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - zdmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - zdmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - azmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - azmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` - azmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` + if [ -e $trackingfile ] && [ $tracknumerrors -eq 0 ] + then # RA - if [ "$ramin" == "$ramax" ] && [ "$ramean" != "0" ] - then - query0=$query0" fRightAscension="$ramean - else - query0=$query0" fRightAscension=NULL" + getfitsstatistics $trackingfile "Ra" $tstart $tstop + if [ "$min" == "$max" ] && [ $good -gt 0 ] + then + query=$query" fRightAscension="$mean + else + query=$query" fRightAscension=NULL" + if [ $good -gt 0 ] + then + printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")." + fi fi # Declination - if [ "$decmin" == "$decmax" ] && [ "$decmean" != "0" ] - then - query0=$query0", fDeclination="$decmean - else - query0=$query0", fDeclination=NULL" + getfitsstatistics $trackingfile "Dec" $tstart $tstop + if [ "$decmin" == "$decmax" ] && [ $good -gt 0 ] + then + query=$query", fDeclination="$mean + else + query=$query", fDeclination=NULL" + if [ $good -gt 0 ] + then + printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")." + fi fi # Zd - if [ "$zdmin" != "" ] && [ "$zdmean" != "0" ] - then - query0=$query0", fZenithDistanceMin="$zdmin - else - query0=$query0", fZenithDistanceMin=NULL" - fi - if [ "$zdmean" != "" ] && [ "$zdmean" != "0" ] - then - query0=$query0", fZenithDistanceMean="$zdmean - else - query0=$query0", fZenithDistanceMean=NULL" - fi - if [ "$zdmax" != "" ] && [ "$zdmean" != "0" ] - then - query0=$query0", fZenithDistanceMax="$zdmax - else - query0=$query0", fZenithDistanceMax=NULL" + getfitsstatistics $trackingfile "Zd" $tstart $tstop + if [ $good -gt 0 ] + then + query=$query", fZenithDistanceMin="$min + query=$query", fZenithDistanceMean="$mean + query=$query", fZenithDistanceMax="$max + else + query=$query", fZenithDistanceMin=NULL" + query=$query", fZenithDistanceMean=NULL" + query=$query", fZenithDistanceMax=NULL" fi # Az - if [ "$azmin" != "" ] && [ "$azmean" != "0" ] - then - query0=$query0", fAzimuthMin="$azmin - else - query0=$query0", fAzimuthMin=NULL" - fi - if [ "$azmean" != "" ] && [ "$azmean" != "0" ] - then - query0=$query0", fAzimuthMean="$azmean - else - query0=$query0", fAzimuthMean=NULL" - fi - if [ "$azmax" != "" ] && [ "$azmean" != "0" ] - then - query0=$query0", fAzimuthMax="$azmax - else - query0=$query0", fAzimuthMax=NULL" - fi + getfitsstatistics $trackingfile "Az" $tstart $tstop + if [ $good -gt 0 ] + then + query=$query", fAzimuthMin="$min + query=$query", fAzimuthMean="$mean + query=$query", fAzimuthMax="$max + else + query=$query", fAzimuthMin=NULL" + query=$query", fAzimuthMean=NULL" + query=$query", fAzimuthMax=NULL" + fi + else + query=$query" fRightAscension=NULL" + query=$query", fDeclination=NULL" + query=$query", fZenithDistanceMin=NULL" + query=$query", fZenithDistanceMean=NULL" + query=$query", fZenithDistanceMax=NULL" + query=$query", fAzimuthMin=NULL" + query=$query", fAzimuthMean=NULL" + query=$query", fAzimuthMax=NULL" fi # get information from trigger - if ! [ -e $triggerratefile ] - then - ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` - ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` - ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` - ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` - if [ "$ratemedian" != "0" ] - then - query0=$query0", fTriggerRateMedian="$ratemedian - else - query0=$query0", fTriggerRateMedian=NULL" - fi + if [ -e $triggerratefile ] && [ $trignumerrors -eq 0 ] + then + #echo " $triggerratefile TriggerRate $tstart $tstop" + getfitsstatistics $triggerratefile "TriggerRate" $tstart $tstop +# if [ "$mjdreftrig" == "" ] +# then +# ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` +# else +# ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` +# fi + if [ $good -gt 0 ] + then + query=$query", fTriggerRateMedian="$median + else + query=$query", fTriggerRateMedian=NULL" + fi + else + query=$query", fTriggerRateMedian=NULL" fi # get information from thresholds - if ! [ -e $thresholdfile ] - then - threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` - threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` - threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` - threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` - if [ "$threshmean" == "0" ] - then - threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` - threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` - threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` - threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` - # is this taking into account the 160 values in a row? - fi - query0=$query0", fThresholdMedian="$threshmedian - fi - -# # get information from bias: Uref -# biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` -# biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` -# biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` -# biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` -# if [ "$biasvoltmean" == "0" ] -# then -# biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` -# biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` -# biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` -# biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` -# # is this taking into account the 160 values in a row? -# fi + if [ -e $thresholdfile ] && [ $treshnumerrors -eq 0 ] + then + getfitsstatistics $thresholdfile "PatchThresh" $tstart $tstop + if [ $good -eq 0 ] + then + getfitsstatistics $thresholdfile "PatchThresh" $tstart2 $tstop + fi + if [ $good -gt 0 ] + then + query=$query", fThresholdMedian="$median + else + query=$query", fThresholdMedian=NULL" + fi + else + query=$query", fThresholdMedian=NULL" + fi # get information from bias: U - if ! [ -e $biasvoltagefile ] - then - biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` - biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` - biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` - biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` - if [ "$biasvoltmean" == "0" ] - then - biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` - biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` - biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` - biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` - # is this taking into account the 160 values in a row? - fi - query0=$query0", fBiasVoltageMedian="$biasvoltmedian + if [ -e $biasvoltagefile ] && [ $biasnumerrors -eq 0 ] + then + getfitsstatistics $biasvoltagefile "U" $tstart $tstop + if [ $good -eq 0 ] + then + getfitsstatistics $biasvoltagefile "U" $tstart2 $tstop + fi + if [ $good -gt 0 ] + then + query=$query", fBiasVoltageMedian="$median + else + query=$query", fBiasVoltageMedian=NULL" + fi + else + query=$query", fBiasVoltageMedian=NULL" fi - query0=$query0" "$queryend - printprocesslog "DEBUG update DB entry with QUERY: "$query0 + # add where condition + query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum + # send query to DB - if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >/dev/null 2>&1 - then - printprocesslog "ERROR insert/update of "$runnumber" "$filenum" to mysql failed." - fi + sendquery >/dev/null done done