#!/bin/bash # missing: # vferify of files # only treat file if it is there # mjd of 1970-01-01 # needed in this script as for 1 day the mjd in the aux files are inconsistent # mjdref=40587 # known: # 2011/11/22 MJDREF in DRIVE empty, Time > 55000 # 2011/11/23 MJDREF in DRIVE not empty, Time > 55000 # 2011/11/24 MJDREF in DRIVE not empty, Time > 15000 # raw files # 2011/11/21 no MJDREF # 2011/11/22 MJDREF # further things: https://www.fact-project.org/logbook/showthread.php?tid=67 # trigger rate has as first value -1, but with using the median it should be fine # option doupdate="yes" # update all entries (needed when new fields have been added) doupdate="no" # fill only entries which are not yet existing (default) source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0 with option doupdate="$doupdate logfile=$runlogpath"/FillAuxLP-"$datetime".log" date >> $logfile function getfitsstatistics() { # $1 filename # $2 colname # $3 tstart # $4 tstop good= min= mean= median= max= tmpfile=`dirname $0`/`basename $1`.tmp echo "ftcopy $1'[col Time]' - | ftstat - | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'" >> $logfile timefromfile=`ftcopy $1'[col Time]' - 2>>$logfile | ftstat - 2>>$logfile | grep 'mean' | grep -E -o [0-9]+[.] | sed -e 's/[.]//g'` if [ $timefromfile -gt 30000 ] then echo "ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile #ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - ftcopy $1'[Time - 40587> '${3}' && Time - 40587< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile else echo "ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat -" >> $logfile #ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - | ftcopy -'[col '${2}']' - | ftstat - ftcopy $1'[Time> '${3}' && Time< '${4}' ][col '${2}';Time]' - 2>>$logfile | ftcopy -'[col '${2}']' - 2>>$logfile | ftstat - 2>>$logfile > $tmpfile fi good=`cat $tmpfile | grep 'good' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` min=`cat $tmpfile | grep 'min' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` mean=`cat $tmpfile | grep 'mean' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` median=`cat $tmpfile | grep 'median' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` max=`cat $tmpfile | grep 'max' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` sigma=`cat $tmpfile | grep 'sigma' | grep -E -o '[-]?[0-9]+[.]?[0-9]*'` if [ "$good" = "" ] then printprocesslog "WARN couldn't get statistics from file $1 for run $date $file" rm $tmpfile finish fi #echo "good: "$good #echo "min: "$min #echo "max: "$max #echo "mean: "$max #echo "median: "$max rm $tmpfile } # setup to use ftools source $HEADAS/headas-init.sh # check if software is available if ! ls $factpath/fitsdump >/dev/null 2>&1 then printprocesslog "ERROR "$factpath"/fitsdump is not available." finish fi # get current hour hour=`date +%k` if [ $hour -le 7 ] || [ $hour -ge 19 ] then dates=( `date +%Y/%m/%d --date="-12hour"` ) else # get last 3, 6 or 9 nights dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \ `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \ # `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \ ) fi #dates=( `find $ziprawdata -mindepth 3 -type d | sort | sed "s/\${ziprawdata_for_sed}//g" | sed -e 's/^\///'` ) #all available dates in /loc_data/zipraw #dates=( "2013/06/13") printprocesslog "INFO processing the following night(s): "${dates[@]} echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1 # do filling of aux data for date in ${dates[@]} do auxdir=$auxdata/$date runnumber=`echo $date | sed -e 's/\///g'` # check if aux files are available from that night if ! [ -d $auxdir ] then printprocesslog "INFO no data available in "$auxdir continue else printprocesslog "INFO processing files in "$auxdir fi # get file numbers from DB # but only for not-corrupted files query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 " printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query filenumbers=( `sendquery $query` ) if [ ${#filenumbers} -eq 0 ] then printprocesslog "INFO No files found in the DB for night "$date continue fi # get daily fits files trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits if ! [ -e $trackingfile ] then printprocesslog "WARN "$trackingfile" not found." else tracknumerrors=`fverify $trackingfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $tracknumerrors -gt 0 ] then printprocesslog "WARN for $trackingfile fverify returned "$tracknumerrors" error(s)." fi fi sourceposfile=$auxdir/$runnumber.DRIVE_CONTROL_SOURCE_POSITION.fits if ! [ -e $sourceposfile ] then printprocesslog "WARN "$sourceposfile" not found." else sourceposnumerrors=`fverify $sourceposfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $sourceposnumerrors -gt 0 ] then printprocesslog "WARN for $sourceposfile fverify returned "$sourceposnumerrors" error(s)." fi fi triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits if ! [ -e $triggerratefile ] then printprocesslog "WARN "$triggerratefile" not found." else trignumerrors=`fverify $triggerratefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $trignumerrors -gt 0 ] then printprocesslog "WARN for $triggerratefile fverify returned "$trignumerrors" error(s)." fi fi thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits if ! [ -e $thresholdfile ] then printprocesslog "WARN "$thresholdfile" not found." else treshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $treshnumerrors -gt 0 ] then printprocesslog "WARN for $thresholdfile fverify returned "$treshnumerrors" error(s)." fi fi biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits if ! [ -e $biasvoltagefile ] then printprocesslog "WARN "$biasvoltagefile" not found." else biasnumerrors=`fverify $biasvoltagefile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $biasnumerrors -gt 0 ] then printprocesslog "WARN for $biasvoltagefile fverify returned "$biasnumerrors" error(s)." fi fi # fill auxiliary information for files for filenum in ${filenumbers[@]} do printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum` echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1 # get information from rawfile rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.gz if ! [ -e $rawfile ] then printprocesslog "ERROR: "$rawfile" not found." continue fi #checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` #if [ "$checkfitsfile" == "" ] #then # numfitserrors=1 # printprocesslog "WARN: "$rawfile" probably corrupted." # continue #fi runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ] then printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF " continue fi # assuming that at least TSTARTI and TSTOPI are consistent #echo $rawfile #echo $tstarti #echo $tstopi #echo $tstartf #echo $tstopf if [ $tstarti -gt 30000 ] then tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l` tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l` else tstart=`echo " $tstarti + $tstartf " | bc -l` tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec tstop=`echo " $tstopi + $tstopf " | bc -l` fi #echo $tstart #echo $tstop #if [ $runnumber -eq 20111123 ] #then # # add mjdref for days were aux files were inconsistent # tstart=`echo " $tstart + $mjdref " | bc -l` # tstart2=`echo " $tstart2 + $mjdref " | bc -l` # tstop=`echo " $tstop + $mjdref " | bc -l` #fi # get information from source_pos file if [ -e $sourceposfile ] && [ $sourceposnumerrors -eq 0 ] then #sourcename=`${factpath}/fitsdump ${sourceposfile} -c Time -c Name --filter='[1]<'${tstop} 2>/dev/null | tail -1 2>&1 | grep -o -E "['][a-zA-Z0-9\ ]+[']" | sed -e "s/'//g"` sourcename=`/home/fact/FACT++.dbg/fitsdump ${sourceposfile} -c Time -c Name --filter='[1]<'${tstop} 2>/dev/null | tail -1 2>&1 | grep -o -E "['][a-zA-Z0-9\+\-\ ]+[']" | sed -e "s/'//g"` #echo $sourcename if [ "$sourcename" == "" ] then printprocesslog "INFO couldn't get sourcename ("$sourcename") from "$sourceposfile" for "$runnumber"_"$filenum else query="SELECT fSourceKey FROM Source WHERE fSourceName='"$sourcename"'" sourcekey=`sendquery` if [ "$sourcename" == "" ] then printprocesslog "WARN couldn't get sourcekey for source "$sourcename" from DB for "$runnumber"_"$filenum fi #echo $sourcekey fi fi # build query to update runinfo in DB query="UPDATE RunInfo SET " # fill source key only if available if ! [ "$sourcekey" = "" ] then query=$query" fSourceKey="$sourcekey", " fi # get information from tracking if [ -e $trackingfile ] && [ $tracknumerrors -eq 0 ] then # RA getfitsstatistics $trackingfile "Ra" $tstart $tstop if [ "$min" == "$max" ] && [ $good -gt 0 ] then query=$query" fRightAscension="$mean else query=$query" fRightAscension=NULL" if [ $good -gt 0 ] then printprocesslog "WARN for $rawfile RA changes within run (min: "$min", max: "$max")." fi fi # Declination getfitsstatistics $trackingfile "Dec" $tstart $tstop if [ "$decmin" == "$decmax" ] && [ $good -gt 0 ] then query=$query", fDeclination="$mean else query=$query", fDeclination=NULL" if [ $good -gt 0 ] then printprocesslog "WARN for $rawfile declination changes within run (min: "$min", max: "$max")." fi fi # Zd getfitsstatistics $trackingfile "Zd" $tstart $tstop if [ $good -gt 0 ] then query=$query", fZenithDistanceMin="$min query=$query", fZenithDistanceMean="$mean query=$query", fZenithDistanceMax="$max else query=$query", fZenithDistanceMin=NULL" query=$query", fZenithDistanceMean=NULL" query=$query", fZenithDistanceMax=NULL" fi # Az getfitsstatistics $trackingfile "Az" $tstart $tstop if [ $good -gt 0 ] then query=$query", fAzimuthMin="$min query=$query", fAzimuthMean="$mean query=$query", fAzimuthMax="$max else query=$query", fAzimuthMin=NULL" query=$query", fAzimuthMean=NULL" query=$query", fAzimuthMax=NULL" fi else query=$query" fRightAscension=NULL" query=$query", fDeclination=NULL" query=$query", fZenithDistanceMin=NULL" query=$query", fZenithDistanceMean=NULL" query=$query", fZenithDistanceMax=NULL" query=$query", fAzimuthMin=NULL" query=$query", fAzimuthMean=NULL" query=$query", fAzimuthMax=NULL" fi # get information from trigger if [ -e $triggerratefile ] && [ $trignumerrors -eq 0 ] then #echo " $triggerratefile TriggerRate $tstart $tstop" getfitsstatistics $triggerratefile "TriggerRate" $tstart $tstop # if [ "$mjdreftrig" == "" ] # then # ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` # ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` # ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` # ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` # else # ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` # ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` # ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` # ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' && Time< '${tstop}' && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` # fi if [ $good -gt 0 ] then query=$query", fTriggerRateMedian="$median else query=$query", fTriggerRateMedian=NULL" fi else query=$query", fTriggerRateMedian=NULL" fi # get information from thresholds if [ -e $thresholdfile ] && [ $treshnumerrors -eq 0 ] then getfitsstatistics $thresholdfile "PatchThresh" $tstart $tstop if [ $good -eq 0 ] then getfitsstatistics $thresholdfile "PatchThresh" $tstart2 $tstop fi if [ $good -gt 0 ] then query=$query", fThresholdMedian="$median else query=$query", fThresholdMedian=NULL" fi else query=$query", fThresholdMedian=NULL" fi # get information from bias: U if [ -e $biasvoltagefile ] && [ $biasnumerrors -eq 0 ] then if [ $runnumber -gt 20120324 ] then value="Uout" else value="U" fi getfitsstatistics $biasvoltagefile $value $tstart $tstop if [ $good -eq 0 ] then getfitsstatistics $biasvoltagefile $value $tstart2 $tstop fi if [ $good -gt 0 ] then query=$query", fBiasVoltageMedian="$median else query=$query", fBiasVoltageMedian=NULL" fi else query=$query", fBiasVoltageMedian=NULL" fi # add where condition query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum #echo $query # send query to DB sendquery >/dev/null done done finish