source: trunk/Mars/hawc/processing/extract-raw-headers.sh@ 20033

#/bin/bash

# Specify the telescope for which to run the script. Replace '1' 
# by $1 if the telesocpe number is supplied as an option to the script
TELESCOPE=1

# This is the directory in which to serach for files
DIR="/data/raw"
# This is the path to the fitsdump executable
FITSDUMP=~/FACT++/build/fitsdump
# File to which the queries are written
OUTPUT="insert-raw.sql"

echo \
"\
CREATE TABLE IF NOT EXISTS RawData
(
  Telescope tinyint UNSIGNED NOT NULL,
  NIGHT int UNSIGNED NOT NULL,
  RUNID mediumint UNSIGNED NOT NULL,
  DATE-OBS datetime(6) NOT NULL,
  DATE-END datetime(6) NOT NULL,
  RUNTYPE tinyint UNSIGNED NOT NULL,
  DRSSTEP tinyint UNSIGNED DEFAULT NULL,
  NROI mediumint UNSIGNED NOT NULL,
  ZNAXIS2 bigint UNSIGNED NOT NULL,
  NTRG int UNSIGNED NOT NULL,
  NTRGMISC int UNSIGNED NOT NULL,
  NTRGPED int UNSIGNED NOT NULL,
  REFCLK float NOT NULL,
  ZRATIO float NOT NULL,
  DAC0 mediumint UNSIGNED NOT NULL,
  DAC1 mediumint UNSIGNED NOT NULL,
  DAC2 mediumint UNSIGNED NOT NULL,
  DAC3 mediumint UNSIGNED NOT NULL,
  DAC4 mediumint UNSIGNED NOT NULL,
  DAC5 mediumint UNSIGNED NOT NULL,
  DAC6 mediumint UNSIGNED NOT NULL,
  DAC7 mediumint UNSIGNED NOT NULL,
  DATE timestamp NOT NULL,
  DRSCALIB tinyint(1) NOT NULL,
  NTRGEXT1 int UNSIGNED NOT NULL,
  NTRGEXT2 int UNSIGNED NOT NULL,
  NTRGLPE int UNSIGNED NOT NULL,
  NTRGLPI int UNSIGNED NOT NULL,
  NTRGTIM int UNSIGNED NOT NULL,
  TSTARTF double NOT NULL,
  TSTARTI mediumint UNSIGNED NOT NULL,
  TSTOPF double NOT NULL,
  TSTOPI mediumint UNSIGNED NOT NULL,
  DNA0 bigint UNSIGNED NOT NULL,
  DNA1 bigint UNSIGNED NOT NULL,
  FWVER0 mediumint UNSIGNED NOT NULL,
  FWVER1 mediumint UNSIGNED NOT NULL,
  PRIMARY KEY (Telescope, NIGHT, RUNID),
  KEY (RUNTYPE)
); 
"\ > ${OUTPUT}


# Loop over all tokens returned by the given bash extension
for SUBDIR in ${DIR}/[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]
do

   # Extract year, month and day from directory name 
   YEAR=`echo  ${SUBDIR} | cut -f6 -d\/`
   MONTH=`echo ${SUBDIR} | cut -f7 -d\/`
   DAY=`echo   ${SUBDIR} | cut -f8 -d\/`

   # Loop over all files in this subdirectory
   for ROOT in ${SUBDIR}/${YEAR}${MONTH}${DAY}*.fits.fz
   do
      FILE=`echo ${ROOT} | cut -f2 -d_`
      echo ${YEAR} ${MONTH} ${DAY} ${FILE} ${ROOT}

      # Extact the header from the fits file with fitsdump, the following chain
      # properly formats the output, replaces T/F with true/false, replaces
      # run-ytpes by numeric types, adds enclosures for SQL variable names where
      # necessary, removes enclosures around hex-numbers and adds commas
      # after all lines except the last one
      RESULT=`${FITSDUMP} -h ${ROOT} \
         | grep 'NIGHT\|RUNID\|TSTART\|TSTOP\|STEP\|DATE\|NROI[^T]\|DAC[0-7]\|DNA[01][^0-9]\|FWVER[01][^0-9]\|DRSCALIB\|NTRG\|NTRG\|ZNAXIS2\|REFCLK\|ZRATIO\|RUNTYPE' \
         | grep -v CHECKSUM \
         | cut -c4-  \
         | cut -f1 -d\/ \
         | sed "s/'drs-pedestal'/1/g" \
         | sed "s/'drs-gain'/2/g" \
         | sed "s/'pedestal'/3/g" \
         | sed "s/'data'/4/g" \
         | sed "s/'custom'/0/g" \
         | sed 's/\ T\ /true/g' \
         | sed 's/\ F\ /false/g' \
         | sed 's/^DATE-END/`DATE-END`/' \
         | sed 's/^DATE-OBS/`DATE-OBS`/' \
         | sed 's/^DATE[^-]/`DATE`/' \
         | sed "s/^\(.*\)'\(0x[0-9a-f]*\)'\(.*\)$/\1\2\3/g" \
         | sed '$!s/$/,/' >> query.txt`

      # count the number of lines in the result
      CNT=`echo ${RESULT} | wc -l`
      
      echo "== $CNT =="
      
      # If the result is not empty write the corresponding insert query to the file
      if [ $CNT -ne 0 ]
      then
          echo "INSERT INTO RawData SET" >> ${OUTPUT}
          echo Telescope=${TELESCOPE},   >> ${OUTPUT}
          echo ${RESULT}                 >> ${OUTPUT}
          echo ";"                       >> ${OUTPUT}                                     
      fi

   done

done