#!/bin/bash # missing: # vferify of files # only treat file if it is there # mjd of 1970-01-01 # needed in this script as for 1 day the mjd in the aux files are inconsistent mjdref=40587 # option doupdate="yes" # update all entries (needed when new fields have been added) doupdate="no" # fill only entries which are not yet existing (default) source `dirname $0`/Sourcefile.sh printprocesslog "INFO starting $0 with option doupdate="$doupdate # setup to use ftools source $HEADAS/headas-init.sh pwfile=`dirname $0`/.pw password=`cat $pwfile 2>/dev/null` if [ "$password" == "" ] then echo "please insert password in $pwfile" printprocesslog "ERROR password for DB access in $pwfile missing" finish fi # check if software is available if ! ls $factpath/fitsdump >/dev/null 2>&1 then printprocesslog "ERROR "$factpath"/fitsdump is not available." finish fi # get last 2 nights dates=( `date +%Y/%m/%d --date="-12hour"` `date +%Y/%m/%d --date="-36hour"` `date +%Y/%m/%d --date="-60hour"` ) #dates=( `date +%Y/%m/%d --date="-12hour"` ) dates=( "2011/12/07" ) # problems with 2011/11/29 dates=( `find /loc_data/zipraw/ -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g'` ) #all available dates in /loc_data/zipraw printprocesslog "INFO processing the following night: "${dates[@]} # do filling of aux data for date in ${dates[@]} do auxdir=$auxdata/$date runnumber=`echo $date | sed -e 's/\///g'` # check if data are available from that night if ! [ -d $auxdir ] then printprocesslog "INFO no data available in "$auxdir continue else printprocesslog "INFO processing files in "$auxdir fi # get daily fits files trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits if ! [ -e $trackingfile ] then printprocesslog "WARN "$trackingfile" not found." fi triggerratefile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits if ! [ -e $triggerratefile ] then printprocesslog "WARN "$triggerratefile" not found." fi thresholdfile=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits if ! [ -e $thresholdfile ] then printprocesslog "WARN "$thresholdfile" not found." fi biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits if ! [ -e $biasvoltagefile ] then printprocesslog "WARN "$biasvoltagefile" not found." fi biascurrentsfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits if ! [ -e $biascurrentsfile ] then printprocesslog "WARN "$biascurrentsfile" not found." fi # get file numbers from DB query1="SELECT fFileNumber from RunInfo WHERE fRunNumber="$runnumber printprocesslog "DEBUG get filenumbers from DB: QUERY: "$query1 filenumbers=( `/usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -s -e "$query1" 2>/dev/null` ) for filenum in ${filenumbers[@]} do printprocesslog "INFO processing file number "$filenum # get information from rawfile rawfile=/loc_data/raw/$date/$runnumber"_"`printf %03d $filenum`.fits if ! [ -e $rawfile ] then printprocesslog "ERROR: "$rawfile" not found." continue fi checkfitsfile=`fverify $rawfile 2>/dev/null | grep '0 error(s)'` if [ "$checkfitsfile" == "" ] then numfitserrors=1 printprocesslog "WARN: "$rawfile" probably corrupted." continue fi runtype=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` mjdrefraw=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` tstarti=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` tstartf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` tstopi=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` tstopf=`$factpath/fitsdump -h -t Events $rawfile 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] then printprocesslog "WARN: "$rawfile" has start or stop time = 0." continue fi tstart=`echo " $tstarti + $tstartf + $mjdrefraw " | bc -l` tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.00011574 " | bc -l` # 10 sec #tstart2=`echo " $tstarti + $tstartf + $mjdrefraw - 0.000023148 " | bc -l` # 2 sec tstop=`echo " $tstopi + $tstopf + $mjdrefraw " | bc -l` # calculate start and stop times # 2011/11/22 MJDREF in DRIVE empty, Time > 55000 # 2011/11/23 MJDREF in DRIVE not empty, Time > 55000 # 2011/11/24 MJDREF in DRIVE not empty, Time > 15000 # raw files # 2011/11/21 no MJDREF # 2011/11/22 MJDREF if [ $runnumber -eq 20111123 ] then # add mjdref for days were aux files were inconsistent tstart=`echo " $tstart + $mjdref " | bc -l` tstart2=`echo " $tstart2 + $mjdref " | bc -l` tstop=`echo " $tstop + $mjdref " | bc -l` fi # build query to update runinfo in DB query0="UPDATE RunInfo SET " queryend=" WHERE fRunNumber="$runnumber" AND fFileNumber="$filenum # get information from tracking if [ -e $trackingfile ] then ramin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` ramax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` ramean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Ra;Time]' - | ftcopy -'[col Ra]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` decmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` decmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` decmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Dec;Time]' - | ftcopy -'[col Dec]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` zdmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` zdmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` zdmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Zd;Time]' - | ftcopy -'[col Zd]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` azmin=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'min' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` azmax=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'max' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` azmean=`ftcopy $trackingfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col Az;Time]' - | ftcopy -'[col Az]' - | ftstat - | grep 'mean' | grep -E -o '[-]*[0-9]+[.]*[0-9]*'` # RA if [ "$ramin" == "$ramax" ] && [ "$ramean" != "0" ] then query0=$query0" fRightAscension="$ramean else query0=$query0" fRightAscension=NULL" fi # Declination if [ "$decmin" == "$decmax" ] && [ "$decmean" != "0" ] then query0=$query0", fDeclination="$decmean else query0=$query0", fDeclination=NULL" fi # Zd if [ "$zdmin" != "" ] && [ "$zdmean" != "0" ] then query0=$query0", fZenithDistanceMin="$zdmin else query0=$query0", fZenithDistanceMin=NULL" fi if [ "$zdmean" != "" ] && [ "$zdmean" != "0" ] then query0=$query0", fZenithDistanceMean="$zdmean else query0=$query0", fZenithDistanceMean=NULL" fi if [ "$zdmax" != "" ] && [ "$zdmean" != "0" ] then query0=$query0", fZenithDistanceMax="$zdmax else query0=$query0", fZenithDistanceMax=NULL" fi # Az if [ "$azmin" != "" ] && [ "$azmean" != "0" ] then query0=$query0", fAzimuthMin="$azmin else query0=$query0", fAzimuthMin=NULL" fi if [ "$azmean" != "" ] && [ "$azmean" != "0" ] then query0=$query0", fAzimuthMean="$azmean else query0=$query0", fAzimuthMean=NULL" fi if [ "$azmax" != "" ] && [ "$azmean" != "0" ] then query0=$query0", fAzimuthMax="$azmax else query0=$query0", fAzimuthMax=NULL" fi fi # get information from trigger if ! [ -e $triggerratefile ] then ratemin=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` ratemax=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` ratemean=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` ratemedian=`ftcopy $triggerratefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF && TriggerRate!=-1][col TriggerRate;Time]' - | ftcopy -'[col TriggerRate]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` if [ "$ratemedian" != "0" ] then query0=$query0", fTriggerRateMedian="$ratemedian else query0=$query0", fTriggerRateMedian=NULL" fi fi # get information from thresholds if ! [ -e $thresholdfile ] then threshmin=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` threshmax=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` threshmean=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` threshmedian=`ftcopy $thresholdfile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` if [ "$threshmean" == "0" ] then threshmin=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` threshmax=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` threshmean=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` threshmedian=`ftcopy $thresholdfile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col PatchThresh;Time]' - | ftcopy -'[col PatchThresh]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` # is this taking into account the 160 values in a row? fi query0=$query0", fThresholdMedian="$threshmedian fi # # get information from bias: Uref # biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` # biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` # biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` # biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` # if [ "$biasvoltmean" == "0" ] # then # biasvoltrefmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` # biasvoltrefmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` # biasvoltrefmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` # biasvoltrefmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' && Time< '${tstop}'][col Uref;Time]' - | ftcopy -'[col Uref]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` # # is this taking into account the 160 values in a row? # fi # get information from bias: U if ! [ -e $biasvoltagefile ] then biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` if [ "$biasvoltmean" == "0" ] then biasvoltmin=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'min' | grep -E -o '[0-9]+[.]?[0-9]*'` biasvoltmax=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'max' | grep -E -o '[0-9]+[.]?[0-9]*'` biasvoltmean=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'mean' | grep -E -o '[0-9]+[.]?[0-9]*'` biasvoltmedian=`ftcopy $biasvoltagefile'[Time> '${tstart2}' - MJDREF && Time< '${tstop}' - MJDREF][col U;Time]' - | ftcopy -'[col U]' - | ftstat - | grep 'median' | grep -E -o '[0-9]+[.]?[0-9]*'` # is this taking into account the 160 values in a row? fi query0=$query0", fBiasVoltageMedian="$biasvoltmedian fi query0=$query0" "$queryend printprocesslog "DEBUG update DB entry with QUERY: "$query0 # send query to DB if ! /usr/bin/mysql -u operator --host=fact01.fact.local --password=$password FactData -e "$query0" >/dev/null 2>&1 then printprocesslog "ERROR insert/update of "$runnumber" "$filenum" to mysql failed." fi done done finish