Changeset 20104 for trunk/Mars


Ignore:
Timestamp:
09/05/21 07:13:17 (3 years ago)
Author:
maslowski
Message:
Added Telescope as command line parameter. And fixed some bugs.
Location:
trunk/Mars/hawc/processing/DiskToDB
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/Mars/hawc/processing/DiskToDB/extract-aux-data2.sh

    r20103 r20104  
    3030# File to which the queries are written
    3131OUTPUT="insert-aux.sql"
    32 TMPOUT="./tmpAuxOut.sql"
    3332# The Mars build directory
    34 readonly MARS="/home/frankm/Dateien/Mars-6.24.00/build4/"
     33readonly MARS="/home/hawc/Desktop/Mars/"
    3534# The path to your Mars/build directory where the processing macros are stored
    36 # MACROS="/home/frankm/Dateien/Mars/hawc/processing"
    37 readonly MACROS="/home/frankm/Dateien/Mars-6.24.00/build/Masterarbeit/Datenbank/HE01/Mexico/Cfiles/"
     35readonly MACROS="/home/hawc/Desktop/Mars-src/hawc/processing/"
    3836readonly PROGRAM=$0
    3937INSERT="INSERT INTO"
    … …  
    9694NIGHT="${4}"
    9795
     96TMPOUT="./tmpAuxOut-${TEL}.sql"
    9897ERRCODE=
    9998
    … …  
    318317
    319318# Delete last comma and replace it with closing parintheses
    320 sed -i '$s/,/)/' "${OUTPUT}"
    321 
    322 # Finish the query file with defining the column names
    323 echo "\
     319if [ -f "${TMPOUT}" ]
     320then
     321        sed -i '$s/,/)/' "${OUTPUT}"
     322
     323        # Finish the query file with defining the column names
     324        echo "\
    324325AS
    325326v(
    … …  
    337338"\ >> ${OUTPUT}
    338339
    339 cat ${TMPOUT} >> ${OUTPUT}
     340else
     341        head -n -4 "${OUTPUT}" > out.sql.new && mv out.sql.new "${OUTPUT}"
     342fi
     343
     344if [ -f "${TMPOUT}" ]
     345then
     346        cat ${TMPOUT} >> ${OUTPUT}
     347fi
    340348
    341349rm -f ${TMPOUT}

  • trunk/Mars/hawc/processing/DiskToDB/extract-raw-header2.sh

    r20103 r20104  
    3232# define variables
    3333OUTPUT=insert-raw.sql
    34 readonly FITSDUMP="/home/frankm/Dateien/FACT++/build/fitsdump"
     34readonly FITSDUMP="/home/hawc/Desktop/FACT++/fitsdump"
    3535readonly PROGRAM=$0
    3636INSERT="INSERT INTO"
    … …  
    108108        local stat="${3}"
    109109
    110         echo "${root} header not valid!"
    111110        echo "UPDATE DataOnDisk SET"            >> ${OUTPUT}
    112111        echo "header=${stat} "                  >> ${OUTPUT}
    … …  
    213212        if [ $EXT -ne 0 ]
    214213        then
     214                echo "${ROOT} Fitsdump failed!"
    215215                SetRawStatus "${ROOT}" ${RUNID} "1"
    216216                continue
    … …  
    234234                SetRawStatus "${ROOT}" ${RUNID} "0"
    235235        else
     236                echo "${ROOT} header not valid!"
    236237                SetRawStatus "${ROOT}" ${RUNID} "2"
    237238        fi

  • trunk/Mars/hawc/processing/DiskToDB/run-callisto2.sh

    r20103 r20104  
    3232readonly STAR=star
    3333# Directory to Mars environment (usually Mars/build)
    34 readonly MARS="/home/frankm/Dateien/Mars-6.24.00/build4/"
     34readonly MARS="/home/hawc/Desktop/Mars/"
    3535# Absolut path to macros
    36 readonly MACROS="/home/frankm/Dateien/Mars-6.24.00/hawc"
     36readonly MACROS="/home/hawc/Desktop/Mars-src/hawc"
    3737readonly PROGRAM=$0
    3838REPLACE=false
    … …  
    139139       --compress \
    140140       | \
    141        while read -r -a LINE
    142        do
    143                # Extract night/runid for data file and calibration file
    144                DATNIGHT=${LINE[0]}
    145                DATRUNID=${LINE[1]}
    146                DRSNIGHT=${LINE[2]}
    147                DRSRUNID=${LINE[3]}
    148 
    149                # Formatting of the file paths and names
    150                DATPATH=${DATNIGHT:0:4}/${DATNIGHT:4:2}/${DATNIGHT:6:2}
    151                DRSPATH=${DRSNIGHT:0:4}/${DRSNIGHT:4:2}/${DRSNIGHT:6:2}
    152 
    153                PREFIX=`printf ${DATNIGHT}_%03d ${DATRUNID}`
    154 
    155                DATNAME=`printf ${DATNIGHT}_%03d.fits.fz ${DATRUNID}`
    156                DRSNAME=`printf ${DRSNIGHT}_%03d.drs.fits ${DRSRUNID}`
    157 
    158                OUT="${DATA}"/../${CALLISTO}/${DATPATH}
    159 
    160                echo DAT=${DATNAME} [${DATPATH}]
    161                echo DRS=${DRSNAME} [${DRSPATH}]
    162 
    163 
    164                if [ ${REPLACE} = true ]
    165                then
    166                        echo "deleting ${OUT}/.${PREFIX}.succsess !!!"
    167                        rm -f "${OUT}"/.${PREFIX}.success
    168                fi
    169 
    170                # Check if not yet successfully processed
    171                # Removing this file can be used to trigger a re-processing
    172                # the next time this script is executed
    173                if [ ! -f "${OUT}/.${PREFIX}.success" ]
    174                then
    175 
    176                        cd ${MARS}
    177 
    178                        mkdir -p "${OUT}"
    179 
    180                        # Trigger reprocessing of the process-fils in the star directory
    181                        rm -f "${DATA}/${STAR}/${DATPATH}/.${PREFIX}.*"
    182 
    183                        # Flag that a process is running
    184                        rm -f "${OUT}"/.${PREFIX}.done
    185                        rm -f "${OUT}"/.${PREFIX}.success
    186 
    187                        touch "${OUT}"/.${PREFIX}.running
    188 
    189                        echo ${DATA}
    190 
    191                        # Execute the calibration and write output to log-file
    192                        root -b -q -l ${MACROS}/callisto.C'("'"${DATA}"/${DATPATH}/${DATNAME}'","'"${DATA}"/${DRSPATH}/${DRSNAME}'", 0 ,"'"${OUT}"'")' \
    193                                2>&1 | tee "${OUT}"/${PREFIX}.log
    194 
    195                        RC=${PIPESTATUS[0]}
    196 
    197 
    198                        # Remember exit status of callisto
    199                        echo RC=${RC} | tee -a "${OUT}"/${PREFIX}.log
    200                        echo ${RC} > "${OUT}"/.${PREFIX}.done
    201                        # Processing is finished
    202                        rm -f "${OUT}"/.${PREFIX}.running
    203 
    204                        # If processing was successfull write corresponding flag
    205                        if [ "${RC}" == "0" ]
    206                        then
     141        while read -r -a LINE
     142        do
     143                # Extract night/runid for data file and calibration file
     144                DATNIGHT=${LINE[0]}
     145                DATRUNID=${LINE[1]}
     146                DRSNIGHT=${LINE[2]}
     147                DRSRUNID=${LINE[3]}
     148
     149                # Formatting of the file paths and names
     150                DATPATH=${DATNIGHT:0:4}/${DATNIGHT:4:2}/${DATNIGHT:6:2}
     151                DRSPATH=${DRSNIGHT:0:4}/${DRSNIGHT:4:2}/${DRSNIGHT:6:2}
     152
     153                PREFIX=`printf ${DATNIGHT}_%03d ${DATRUNID}`
     154
     155                DATNAME=`printf ${DATNIGHT}_%03d.fits.fz ${DATRUNID}`
     156                DRSNAME=`printf ${DRSNIGHT}_%03d.drs.fits ${DRSRUNID}`
     157
     158                OUT="${DATA}"/../${CALLISTO}/${DATPATH}
     159
     160                echo DAT=${DATNAME} [${DATPATH}]
     161                echo DRS=${DRSNAME} [${DRSPATH}]
     162
     163
     164                if [ ${REPLACE} = true ]
     165                then
     166                        echo "deleting ${OUT}/.${PREFIX}.succsess !!!"
     167                        rm -f "${OUT}"/.${PREFIX}.success
     168                fi
     169
     170                # Check if not yet successfully processed
     171                # Removing this file can be used to trigger a re-processing
     172                # the next time this script is executed
     173                if [ ! -f "${OUT}/.${PREFIX}.success" ]
     174                then
     175
     176                        cd ${MARS}
     177
     178                        mkdir -p "${OUT}"
     179
     180                        # Trigger reprocessing of the process-fils in the star directory
     181                        rm -f "${DATA}/${STAR}/${DATPATH}/.${PREFIX}.*"
     182
     183                        # Flag that a process is running
     184                        rm -f "${OUT}"/.${PREFIX}.done
     185                        rm -f "${OUT}"/.${PREFIX}.success
     186
     187                        touch "${OUT}"/.${PREFIX}.running
     188
     189                        echo ${DATA}
     190
     191                        RC=0
     192                        # Execute the calibration and write output to log-file
     193                        root -b -q -l ${MACROS}/callisto.C'("'"${DATA}"/${DATPATH}/${DATNAME}'","'"${DATA}"/${DRSPATH}/${DRSNAME}'", 0 ,"'"${OUT}"'")' \
     194                               2>&1 | tee "${OUT}"/${PREFIX}.log || RC=1
     195
     196                        # Remember exit status of callisto
     197                        echo RC=${RC} | tee -a "${OUT}"/${PREFIX}.log
     198                        echo ${RC} > "${OUT}"/.${PREFIX}.done
     199                        # Processing is finished
     200                        rm -f "${OUT}"/.${PREFIX}.running
     201
     202                        # If processing was successfull write corresponding flag
     203                        if [ "${RC}" == "0" ]
     204                        then
    207205                                touch "${OUT}"/.${PREFIX}.success
    208                        else
    209                                 UpdateStatus ${TEL} ${NIGHT} ${DATRUNID} 1
    210                        fi
    211 
    212                        cd -
    213 
    214                        echo "-----End of first iteration-----"
    215                else
    216                        echo Skipped.
    217                fi
    218 
    219        done
     206                        fi
     207
     208                        cd -
     209
     210                        echo "-----End of first iteration-----"
     211                else
     212                        echo Skipped.
     213                fi
     214
     215        done

  • trunk/Mars/hawc/processing/DiskToDB/run-root2sql2.sh

    r20103 r20104  
    2828
    2929# Path to the executable
    30 readonly ROOT2SQL="/home/frankm/Dateien/FACT++/build/root2sql"
     30readonly ROOT2SQL="/home/hawc/Desktop/FACT++/root2sql"
    3131readonly PROGRAM=${0}
    3232
    … …  
    128128        fi
    129129
    130         mysql --defaults-file=${CRED} \
    131                 --compress \
    132                 -e "UPDATE DataOnDisk \
    133                 SET DataOnDisk.${col} = 6 \
    134                 WHERE DataOnDisk.Telescope = ${tel} \
    135                 AND DataOnDisk.NIGHT = ${night} \
    136                 AND DataOnDisk.calibration != 0;"
    137 
     130        # Set col to 3 if the file is a DRS run
    138131        mysql --defaults-file=${CRED} \
    139132                --compress \

  • trunk/Mars/hawc/processing/DiskToDB/run-scripts.sh

    r20103 r20104  
    3636
    3737
     38
     39function LockExit()
     40{
     41        echo "ERROR: One instance of ${SCRPTN} already running" >&2
     42        exit 1
     43}
     44
     45function DescExit()
     46{
     47        echo "ERROR: Filedescriptor 200 already taken" >&2
     48        exit 1
     49}
     50
     51function ErrExit()
     52{
     53        echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
     54        exit 1
     55}
     56
     57function StrgCExit()
     58{
     59        echo " "
     60        echo "$0 was forcefully terminated" >&2
     61        exit 1
     62}
     63
     64trap ErrExit ERR
     65trap StrgCExit INT
     66
     67# Telescope for which to process
     68readonly TEL="${1}"
     69if [ ${TEL} = "1" ]
     70then
     71        echo "Processing for Telescope 1"
     72elif [ ${TEL} = "2" ]
     73then
     74        echo "Processing for Telescope 2"
     75else
     76        echo "No valid Telescope selected!" >&2
     77        exit 1
     78fi
     79
    3880# Filename
    3981readonly SCRPTN=`basename $0 .sh`
    4082
    4183# For lockfile
    42 readonly LOCK="/tmp/${SCRPTN}.lock"
    43 
    44 function LockExit()
    45 {
    46         echo "ERROR: One instance of ${SCRPTN} already running" >&2
    47         exit 1
    48 }
    49 
    50 function DescExit()
    51 {
    52         echo "ERROR: Filedescriptor 200 already taken" >&2
    53         exit 1
    54 }
    55 
    56 function ErrExit()
    57 {
    58         echo "ERROR: Line `caller`: ${BASH_COMMAND}" >&2
    59         exit 1
    60 }
    61 
    62 function StrgCExit()
    63 {
    64         echo " "
    65         echo "$0 was forcefully terminated" >&2
    66         exit 1
    67 }
    68 
    69 trap ErrExit ERR
    70 trap StrgCExit INT
    71 
    72 exec 200>${LOCK} || DescExit
    73 flock -n 200 || LockExit
     84readonly LOCK="/tmp/${SCRPTN}${TEL}.lock"
     85
     86eval "exec 20${TEL}>${LOCK}" || DescExit
     87flock -n "20${TEL}" || LockExit
     88
     89cd /home/hawc/Desktop/DiskToDB/
    7490
    7591# For logfile
    7692mkdir -p log
    7793readonly DATE=`date +%Y%m%d_%H%M`
    78 readonly LOG="./log/${SCRPTN}_${DATE}.log"
    79 # exec 3>&1 1>>${LOG}
    80 # exec 4>>${LOG}
    81 exec 3>&1
    82 exec 4>&1
     94readonly LOG="./log/${SCRPTN}_${DATE}_${TEL}.log"
     95exec 3>&1 1>>${LOG}
     96exec 4>>${LOG}
     97exec 5>&2 2>>${LOG}
     98# exec 3>&1
     99# exec 4>&1
    83100
    84101echo "Start processing data"
    … …  
    188205}
    189206
    190 
    191207# Define variables
    192 # Telescope for which to process
    193 readonly TEL=1
    194 # Directory in which the raw data is located
    195 readonly DIR='/run/media/frankm/Neuer Datenträger/data/DATA/HE0'${TEL}
     208# Directory in which the raw directory is located
     209readonly DIR="/media/DATA/HAWCsEYE0${TEL}/"
    196210# Credentials for the SQL database
    197 readonly CRED='../credentials-read-only.cnf'
     211readonly CRED='../Credentials/credentials-read-only.cnf'
    198212# Credentials for root2sql for callisto
    199 readonly CALLISTORC='../root2sql-callisto.rc'
     213readonly CALLISTORC='../Credentials/root2sql-callisto.rc'
    200214# Credentials for root2sql for star
    201 readonly STARRC='../root2sql-star.rc'
     215readonly STARRC='../Credentials/root2sql-star.rc'
     216# ROOT init path
     217readonly RPATH='../root-6.24.02/bin/thisroot.sh'
     218
     219echo "Processing data from ${DIR}"
    202220
    203221# Check if variable paths point to something sensible
    … …  
    206224DoesFileExists "${CALLISTORC}"
    207225DoesFileExists "${STARRC}"
     226DoesFileExists "${RPATH}"
    208227
    209228# SQL outputs
    210229mkdir -p ./queries
    211 readonly ONDISK='./queries/insert-OnDisk.sql'
    212 readonly RAW='./queries/insert-raw.sql'
    213 readonly CALIB='./queries/find-calibration2.sql'
    214 readonly AUX='./queries/insert-aux.sql'
     230readonly ONDISK="./queries/insert-OnDisk-${TEL}.sql"
     231readonly RAW="./queries/insert-raw-${TEL}.sql"
     232readonly CALIB="./queries/find-calibration2-${TEL}.sql"
     233readonly AUX="./queries/insert-aux-${TEL}.sql"
     234
     235# Init ROOT
     236set +o nounset
     237. "${RPATH}"
     238set -o nounset
    215239
    216240# Create Database of existing files
    … …  
    247271echo "Updating database"
    248272
    249 NIGHTS=20201111
    250 
    251273echo "Nights to process:"
    252274echo "${NIGHTS}"

  • trunk/Mars/hawc/processing/DiskToDB/run-star2.sh

    r20103 r20104  
    3232readonly STAR=star
    3333# Directory to Mars environment (usually Mars/build)
    34 readonly MARS="/home/frankm/Dateien/Mars-6.24.00/build4/"
     34readonly MARS="/home/hawc/Desktop/Mars/"
    3535# Absolut path to macros
    36 readonly MACROS="/home/frankm/Dateien/Mars-6.24.00/hawc"
     36readonly MACROS="/home/hawc/Desktop/Mars-src/hawc"
    3737readonly PROGRAM=$0
    3838REPLACE=false
    … …  
    110110"\
    111111SELECT
    112         NIGHT, RUNID
     112        Calibration.NIGHT,
     113        Calibration.RUNID
    113114FROM
    114         Calibration
     115        Calibration
     116INNER JOIN
     117        DataOnDisk
     118ON
     119        DataOnDisk.Telescope = Calibration.Telescope
     120AND
     121        DataOnDisk.NIGHT = Calibration.NIGHT
     122AND
     123        DataOnDisk.RUNID = Calibration.RUNID
    115124WHERE
    116         Telescope=${TEL}
     125        Calibration.Telescope = ${TEL}
     126AND     
     127        Calibration.NIGHT = ${NIGHT}
    117128AND
    118         NIGHT=${NIGHT}
     129        DataOnDisk.callisto = 0
    119130ORDER BY
    120         NIGHT, RUNID\
     131        Calibration.NIGHT, Calibration.RUNID\
    121132"\
    122133 |  mysql \
    … …  
    165176                        touch "${OUT}"/.${PREFIX}.running
    166177
     178                        RC=0
     179                        # set +o pipefail
    167180                        # Execute image parameter calculation and write log-file
    168181                        root -b -q -l ${MACROS}/star.C'("'"${IN}"/${DATNAME}'","'"${OUT}"'")' \
    169                                 2>&1 | tee "${OUT}"/${PREFIX}.log
    170 
    171                         # Remember exit status of callisto
    172                         RC=${PIPESTATUS[0]}
     182                                2>&1 | tee "${OUT}"/${PREFIX}.log || RC=1
     183
     184                        # RC=$?
     185                        # set -o pipefail
     186
     187                        # Remember exit status of star
     188                        # RC=${PIPESTATUS[0]}
    173189
    174190                        echo RC=${RC} | tee -a "${OUT}"/${PREFIX}.log
    … …  
    182198                        then
    183199                                touch "${OUT}"/.${PREFIX}.success
    184                         else
    185                                 UpdateStatus ${TEL} ${NIGHT} ${DATRUNID} 1
    186200                        fi
    187201

  • trunk/Mars/hawc/processing/DiskToDB/update-file-table2.sh

    r20103 r20104  
    169169) SELECT * FROM DataOnDisk
    170170        WHERE (Telescope, NIGHT, RUNID, ISDRSFILE) NOT IN
    171         (SELECT Telescope, NIGHT, RUNID, ISDRSFILE FROM DataOnDisk_tmp);
     171        (SELECT Telescope, NIGHT, RUNID, ISDRSFILE FROM DataOnDisk_tmp)
     172        AND Telescope=${TEL};
    172173
    173 UPDATE DataOnDisk SET star = NULL WHERE NIGHT IN
    174         (SELECT NIGHT FROM TableNotDisk);
     174UPDATE DataOnDisk SET header = NULL WHERE NIGHT IN
     175        (SELECT NIGHT FROM TableNotDisk)
     176        AND Telescope=${TEL};
    175177
    176178DELETE FROM DataOnDisk
    177179        WHERE (Telescope, NIGHT, RUNID, ISDRSFILE) NOT IN
    178         (SELECT Telescope, NIGHT, RUNID, ISDRSFILE FROM DataOnDisk_tmp);
     180        (SELECT Telescope, NIGHT, RUNID, ISDRSFILE FROM DataOnDisk_tmp)
     181        AND Telescope=${TEL};
    179182
    180183${INSERT} DataOnDisk (SELECT * FROM DataOnDisk_tmp);

  • trunk/Mars/hawc/processing/DiskToDB/write-calibration-query.sh

    r20103 r20104  
    9797UPDATE DataOnDisk
    9898SET DataOnDisk.calibration = 6
    99 WHERE DataOnDisk.header != 0;
     99WHERE DataOnDisk.header != 0
     100AND NIGHT=${NIGHT}
     101AND Telescope=${TEL};
    100102
    101103CREATE TEMPORARY TABLE Calibration_tmp
    … …  
    205207UPDATE DataOnDisk
    206208SET DataOnDisk.calibration = 0
    207 WHERE DataOnDisk.header = 0;
     209WHERE DataOnDisk.header = 0
     210AND NIGHT=${NIGHT}
     211AND Telescope=${TEL};
    208212
    209213-- Set Status to 2 if there was no drs file with step = 1 in the NIGHT
    210214UPDATE DataOnDisk
    211215SET DataOnDisk.calibration = 2
    212 WHERE EXISTS(SELECT * FROM Calibration WHERE NIGHT=${NIGHT}) = 0;
     216WHERE EXISTS(
     217        SELECT * FROM Calibration
     218        WHERE NIGHT=${NIGHT}
     219        AND Telescope=${TEL}
     220        ) = 0;
    213221"\ >> ${OUTPUT}
Note: See TracChangeset for help on using the changeset viewer.