Index: trunk/DataCheck/Processing/FillAuxData.sh =================================================================== --- trunk/DataCheck/Processing/FillAuxData.sh (revision 13040) +++ trunk/DataCheck/Processing/FillAuxData.sh (revision 13040) @@ -0,0 +1,363 @@ +#!/bin/bash + +# missing: +# vferify of files +# only treat file if it is there + +# mjd of 1970-01-01 +# needed in this script as for 1 day the mjd in the aux files are inconsistent +# mjdref=40587 + +# known: +# 2011/11/22 MJDREF in DRIVE empty, Time > 55000 +# 2011/11/23 MJDREF in DRIVE not empty, Time > 55000 +# 2011/11/24 MJDREF in DRIVE not empty, Time > 15000 +# raw files +# 2011/11/21 no MJDREF +# 2011/11/22 MJDREF +# further things: https://www.fact-project.org/logbook/showthread.php?tid=67 + +# trigger rate has as first value -1, but with using the median it should be fine + +# option +doupdate="yes" # update all entries (needed when new fields have been added) +doupdate="no" # fill only entries which are not yet existing (default) + +source `dirname $0`/Sourcefile.sh +printprocesslog "INFO starting $0 with option doupdate="$doupdate + +logfile=$runlogpath"/FillAuxLP-"$datetime".log" +date >> $logfile + +function getfitsstatistics() +{ + # $1 filename + # $2 colname + # $3 tstart + # $4 tstop + good= + min= + mean= + median= + max= + tmpfile=`dirname $0`/`basename $1`.tmp + echo "ftcopy $1'[col Time]' - | ftstat - | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'" >> $logfile + timefromfile=`ftcopy $1'[col Time]' - 2>>$logfile | ftstat - 2>>$logfile | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'` + if [ $timefromfile -gt 30000 ] + then + echo "ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile + #ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - + ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile + else + echo "ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile + #ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - + ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile + fi + good=`cat $tmpfile | grep 'good' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + min=`cat $tmpfile | grep 'min' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + mean=`cat $tmpfile | grep 'mean' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + median=`cat $tmpfile | grep 'median' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + max=`cat $tmpfile | grep 'max' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` + if [ "$good" = "" ] + then + printprocesslog "WARN couldn't get statistics from file $1 for run $date $file" + rm $tmpfile + finish + fi + #echo "good: "$good + #echo "min: "$min + #echo "max: "$max + #echo "mean: "$max + #echo "median: "$max + rm $tmpfile +} + +# setup to use ftools +source $HEADAS/headas-init.sh + +# check if software is available +if ! ls $factpath/fitsdump >/dev/null 2>&1 +then + printprocesslog "ERROR "$factpath"/fitsdump is not available." + finish +fi + + +# get last 3 nights +dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` ) +#dates=( `find $ziprawdata -mindepth 3 -type d | sort | sed "s/\${ziprawdata_for_sed}//g" | sed -e 's/^\///'` ) #all available dates in /loc_data/zipraw +printprocesslog "INFO processing the following night(s): "${dates[@]} +echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1 + +# do filling of aux data +for date in ${dates[@]} +do + auxdir=$auxdata/$date + runnumber=`echo $date | sed -e 's/\///g'` + + # check if aux files are available from that night + if ! [ -d $auxdir ] + then + printprocesslog "INFO no data available in "$auxdir + continue + else + printprocesslog "INFO processing files in "$auxdir + fi + + # get file numbers from DB + # but only for not-corrupted files + query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 " + printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query + filenumbers=( `sendquery $query` ) + if [ ${#filenumbers} -eq 0 ] + then + printprocesslog "INFO No files found in the DB for night "$date + continue + fi + + # get daily fits files + trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits + if ! [ -e $trackingfile ] + then + printprocesslog "WARN "$trackingfile" not found." + else + tracknumerrors=`fverify $trackingfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $tracknumerrors -gt 0 ] + then + printprocesslog "WARN for $trackingfile fverify returned "$tracknumerrors" error(s)." + fi + fi + + triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits + if ! [ -e $triggerratefile ] + then + printprocesslog "WARN "$triggerratefile" not found." + else + trignumerrors=`fverify $triggerratefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $trignumerrors -gt 0 ] + then + printprocesslog "WARN for $triggerratefile fverify returned "$trignumerrors" error(s)." + fi + fi + + thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits + if ! [ -e $thresholdfile ] + then + printprocesslog "WARN "$thresholdfile" not found." + else + treshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $treshnumerrors -gt 0 ] + then + printprocesslog "WARN for $thresholdfile fverify returned "$treshnumerrors" error(s)." + fi + fi + + biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits + if ! [ -e $biasvoltagefile ] + then + printprocesslog "WARN "$biasvoltagefile" not found." + else + biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` + if [ $biasnumerrors -gt 0 ] + then + printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)." + fi + fi + + # fill auxiliary information for files + for filenum in ${filenumbers[@]} + do + printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum` + echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1 + # get information from rawfile + rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.gz + if ! [ -e $rawfile ] + then + printprocesslog "ERROR: "$rawfile" not found." + continue + fi + #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` + #if [ "$checkfitsfile" == "" ] + #then + # numfitserrors=1 + # printprocesslog "WARN: "$rawfile" probably corrupted." + # continue + #fi + runtype=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` + mjdrefraw=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + tstarti=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + tstartf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + tstopi=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` + tstopf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` + if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ] + then + printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF " + continue + fi + # assuming that at least TSTARTI and TSTOPI are consistent + #echo $rawfile + #echo $tstarti + #echo $tstopi + #echo $tstartf + #echo $tstopf + if [ $tstarti -gt 30000 ] + then + tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l` + tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec + #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec + tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l` + else + tstart=`echo " $tstarti + $tstartf " | bc -l` + tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec + #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec + tstop=`echo " $tstopi + $tstopf " | bc -l` + fi + #echo $tstart + #echo $tstop + #if [ $runnumber -eq 20111123 ] + #then + # # add mjdref for days were aux files were inconsistent + # tstart=`echo " $tstart + $mjdref " | bc -l` + # tstart2=`echo " $tstart2 + $mjdref " | bc -l` + # tstop=`echo " $tstop + $mjdref " | bc -l` + #fi + + # build query to update runinfo in DB + query="UPDATE RunInfo SET " + + # get information from tracking + if [ -e $trackingfile ] && [ $tracknumerrors -eq 0 ] + then + # RA + getfitsstatistics $trackingfile "Ra" $tstart $tstop + if [ "$min" == "$max" ] && [ $good -gt 0 ] + then + query=$query" fRightAscension="$mean + else + query=$query" fRightAscension=NULL" + if [ $good -gt 0 ] + then + printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")." + fi + fi + # Declination + getfitsstatistics $trackingfile "Dec" $tstart $tstop + if [ "$decmin" == "$decmax" ] && [ $good -gt 0 ] + then + query=$query", fDeclination="$mean + else + query=$query", fDeclination=NULL" + if [ $good -gt 0 ] + then + printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")." + fi + fi + # Zd + getfitsstatistics $trackingfile "Zd" $tstart $tstop + if [ $good -gt 0 ] + then + query=$query", fZenithDistanceMin="$min + query=$query", fZenithDistanceMean="$mean + query=$query", fZenithDistanceMax="$max + else + query=$query", fZenithDistanceMin=NULL" + query=$query", fZenithDistanceMean=NULL" + query=$query", fZenithDistanceMax=NULL" + fi + # Az + getfitsstatistics $trackingfile "Az" $tstart $tstop + if [ $good -gt 0 ] + then + query=$query", fAzimuthMin="$min + query=$query", fAzimuthMean="$mean + query=$query", fAzimuthMax="$max + else + query=$query", fAzimuthMin=NULL" + query=$query", fAzimuthMean=NULL" + query=$query", fAzimuthMax=NULL" + fi + else + query=$query" fRightAscension=NULL" + query=$query", fDeclination=NULL" + query=$query", fZenithDistanceMin=NULL" + query=$query", fZenithDistanceMean=NULL" + query=$query", fZenithDistanceMax=NULL" + query=$query", fAzimuthMin=NULL" + query=$query", fAzimuthMean=NULL" + query=$query", fAzimuthMax=NULL" + fi + + # get information from trigger + if [ -e $triggerratefile ] && [ $trignumerrors -eq 0 ] + then + #echo " $triggerratefile TriggerRate $tstart $tstop" + getfitsstatistics $triggerratefile "TriggerRate" $tstart $tstop +# if [ "$mjdreftrig" == "" ] +# then +# ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` +# else +# ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` +# ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` +# fi + if [ $good -gt 0 ] + then + query=$query", fTriggerRateMedian="$median + else + query=$query", fTriggerRateMedian=NULL" + fi + else + query=$query", fTriggerRateMedian=NULL" + fi + + # get information from thresholds + if [ -e $thresholdfile ] && [ $treshnumerrors -eq 0 ] + then + getfitsstatistics $thresholdfile "PatchThresh" $tstart $tstop + if [ $good -eq 0 ] + then + getfitsstatistics $thresholdfile "PatchThresh" $tstart2 $tstop + fi + if [ $good -gt 0 ] + then + query=$query", fThresholdMedian="$median + else + query=$query", fThresholdMedian=NULL" + fi + else + query=$query", fThresholdMedian=NULL" + fi + + # get information from bias: U + if [ -e $biasvoltagefile ] && [ $biasnumerrors -eq 0 ] + then + getfitsstatistics $biasvoltagefile "U" $tstart $tstop + if [ $good -eq 0 ] + then + getfitsstatistics $biasvoltagefile "U" $tstart2 $tstop + fi + if [ $good -gt 0 ] + then + query=$query", fBiasVoltageMedian="$median + else + query=$query", fBiasVoltageMedian=NULL" + fi + else + query=$query", fBiasVoltageMedian=NULL" + fi + + # add where condition + query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum + + # send query to DB + sendquery >/dev/null + done +done + +finish + +