#!/bin/bash # option doupdate="yes" # update all entries (needed when new fields have been added) doupdate="no" # fill only entries which are not yet existing (default) source `dirname $0`/../Sourcefile.sh printprocesslog "INFO starting $0 with option doupdate="$doupdate logfile=$runlogpath"/FillThresholds-"$datetime".log" date >> $logfile # setup to use ftools source $HEADAS/headas-init.sh # check if software is available if ! ls $factpath/fitsdump >/dev/null 2>&1 then printprocesslog "ERROR "$factpath"/fitsdump is not available." finish fi # get last 3, 6 or 9 nights dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` \ # `date +%Y/%m/%d --date="-84hour"` `date +%Y/%m/%d --date="-108hour"` `date +%Y/%m/%d --date="-132hour"` \ # `date +%Y/%m/%d --date="-156hour"` `date +%Y/%m/%d --date="-180hour"` `date +%Y/%m/%d --date="-204hour"` \ ) #dates=( `find -L $ziprawdata -mindepth 3 -type d | sort -r | sed "s/\${ziprawdata_for_sed}//g" | sed -e 's/^\///'` ) #all available dates in /loc_data/zipraw #dates=( `find -L $rawdata -mindepth 3 -type d | sort -r | sed "s/\${rawdata_for_sed}//g" | sed -e 's/^\///'` ) #all available dates in /fact/raw #dates=( "2013/01/31" ) printprocesslog "INFO processing the following night(s): "${dates[@]} echo `date`": processing the following night(s): "${dates[@]} >> $logfile 2>&1 echo ${dates[@]} cd $mars # do filling of aux data for date in ${dates[@]} do auxdir=$auxdata/$date runnumber=`echo $date | sed -e 's/\///g'` echo $auxdir # check if aux files are available from that night if ! [ -d $auxdir ] then printprocesslog "INFO no data available in "$auxdir continue else printprocesslog "INFO processing files in "$auxdir fi thresholdfile=$auxdir/$runnumber.RATE_CONTROL_THRESHOLD.fits ls $thresholdfile if ! [ -e $thresholdfile ] then printprocesslog "WARN "$thresholdfile" not found." continue else threshnumerrors=`fverify $thresholdfile 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $threshnumerrors -gt 0 ] then printprocesslog "WARN for $thresholdfile fverify returned "$threshnumerrors" error(s)." fi fi thresholdfile2=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits ls $thresholdfile2 if ! [ -e $thresholdfile2 ] then printprocesslog "WARN "$thresholdfile2" not found." continue else threshnumerrors2=`fverify $thresholdfile2 2>/dev/null | grep -o '[0-9][ ]error(s)' | grep -E -o '[0-9]'` if [ $threshnumerrors2 -gt 0 ] then printprocesslog "WARN for $thresholdfile2 fverify returned "$threshnumerrors2" error(s)." fi fi # get file numbers from DB # but only for not-corrupted files query="SELECT fRunID from RunInfo WHERE fNight="$runnumber" AND fFitsFileErrors=0 " if [ "$doupdate" = "no" ] then query=$query" AND ISNULL(fThresholdMedMean) AND ISNULL(fThresholdMinSet) " fi printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query filenumbers=( `sendquery $query` ) if [ ${#filenumbers} -eq 0 ] then printprocesslog "INFO No files found in the DB for night "$date continue fi # fill auxiliary information for files for filenum in ${filenumbers[@]} do printprocesslog "INFO processing file number "$runnumber"_"`printf %03d $filenum` echo `date`": processing file number "$runnumber"_"`printf %03d $filenum` >> $logfile 2>&1 # get information from rawfile rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.gz if ! [ -e $rawfile ] then printprocesslog "ERROR: "$rawfile" not found." continue fi runtype=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` mjdrefraw=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` tstarti=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` tstartf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` tstopi=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` tstopf=`$factpath/fitsdump -h $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ] then printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF " continue fi # assuming that at least TSTARTI and TSTOPI are consistent #echo $rawfile #echo $tstarti #echo $tstopi #echo $tstartf #echo $tstopf if [ $tstarti -gt 30000 ] then tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l` tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l` # 10 sec #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l` else tstart=`echo " $tstarti + $tstartf " | bc -l` tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l` # 10 sec #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec tstop=`echo " $tstopi + $tstopf " | bc -l` fi # build query to update runinfo in DB query="UPDATE RunInfo SET " # get information if [ -e $threshcurrentsfile ] && [ $threshnumerrors -eq 0 ] then thresholds1=( `root -q -b -l fact/threshold.C\("\"$thresholdfile2\""\,$tstart\,$tstop\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` ) #root -q -b -l fact/threshold.C\("\"$thresholdfile2\""\,$tstart\,$tstop\) | grep "result" echo ${thresholds1[@]} if [ "${thresholds1[0]}" == "" ] then query=$query"fThresholdMedMean=NULL" else query=$query"fThresholdMedMean="${thresholds1[0]} fi if [ "${thresholds1[1]}" == "" ] then query=$query", fThresholdMedRms=NULL" else query=$query", fThresholdMedRms="${thresholds1[1]} fi if [ "${thresholds1[2]}" == "" ] then query=$query", fThresholdMax=NULL" else query=$query", fThresholdMax="${thresholds1[2]} fi if [ "${thresholds1[3]}" == "" ] then query=$query", fThresholdAvgMean=NULL" else query=$query", fThresholdAvgMean="${thresholds1[3]} fi else query=$query" fThresholdMedMean=NULL" query=$query", fThresholdMedRms=NULL" query=$query", fThresholdMax=NULL" query=$query", fThresholdAvgMean=NULL" fi # get information if [ -e $threshcurrentsfile2 ] && [ $threshnumerrors2 -eq 0 ] then thresholds2=( `root -q -b -l fact/lastth.C\("\"$thresholdfile\""\,$tstart\) | grep "result" | grep -E -o '[0-9]+[.]?[0-9]*'` ) #root -q -b -l fact/lastth.C\("\"$thresholdfile\""\,$tstart\) | grep "result" echo ${thresholds2[@]} if [ "${thresholds2[0]}" == "" ] then query=$query", fThresholdMinSet=NULL" else query=$query", fThresholdMinSet="${thresholds2[0]} fi if [ "${thresholds2[1]}" == "" ] then query=$query", fThresholdMinTimeDiff=NULL" else query=$query", fThresholdMinTimeDiff="${thresholds2[1]} fi else query=$query", fThresholdMinSet=NULL" query=$query", fThresholdMinTimeDiff=NULL" fi # add where condition query=$query" WHERE fNight="$runnumber" AND fRunID="$filenum echo $query # send query to DB sendquery >/dev/null done done finish