Changeset 16677


Ignore:
Timestamp:
06/03/13 22:38:39 (11 years ago)
Author:
Daniela Dorner
Message:
added loop over dates and usage of sge
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/QuickLook/Step2a.sh

    r15521 r16677  
    77then
    88   # get date (before 18h there is no new data to be processed)
    9    datepath=`date --date="-18HOUR" +%Y/%m/%d`
     9   datepaths=( `date --date="-18HOUR" +%Y/%m/%d` )
     10   datepaths=( `find -L $datapath/star -mindepth 3 -type d | sort -r | sed "s/\${datapath_for_sed}\/star//g" | sed -e 's/^\///'` ) #all available dates in star-directory
    1011else
    11    datepath=$1
     12   datepaths=( $1 )
    1213fi
    13 printprocesslog "INFO processing "$datepath
    14 #echo "INFO processing "$datepath
    15 night=`echo $datepath | sed -e 's/\///g'`
    1614
    17 numganymeds=0
     15echo ${datepaths[@]}
    1816
    19 # getting all image files for this night
    20 printprocesslog "INFO get file list for night "$datepath
    21 files=( `find /loc_data/analysis/star/$datepath -type f -name '*_I.root' 2>/dev/null | sort` )
    22 if [ ${#files[@]} -eq 0 ]
    23 then
    24    printprocesslog "INFO no image files available for night "$datepath
    25    finish
    26 fi
    27 # write data set file
    28 #dspath=$anapath/$source/datasets`printf %03d $min`min/$datepath
    29 #dspath=$anapath/$source/datasets_run/$datepath
    30 dspath=$anapath/datasets_run/$datepath
    31 makedir $dspath
    32 
    33 for (( i=1 ; i<=${#files[@]} ; i++ ))
     17for datepath in ${datepaths[@]}
    3418do
    35    night=`basename ${files[$i-1]} | cut -d_ -f1`
    36    run=`basename ${files[$i-1]} | cut -d_ -f2`
    37    printprocesslog "DEBUG night: "$night" run: "$run
    38    dsfile=$dspath"/dataset"$night"_"$run".txt"
    39    #echo $dsfile
    40    printprocesslog "INFO writing dataset file "$dsfile
    41    # write dataset only if merpp-log is available to make sure that star-file is complete
    42    merpplog=`echo ${files[$i-1]} | sed -e 's/_I.root/-merpp.log/'`
    43    stillrunning=`find $merpplog -mmin -1 2>/dev/null`
    44    if [ -e $merpplog ] && [ "$stillrunning" == "" ]
    45    then
    46       echo `dirname ${files[$i-1]}`" "`basename ${files[$i-1]}` > $dsfile
    47    else
    48       printprocesslog "INFO merpp still running - wait with file "${files[$i-1]}
     19   printprocesslog "INFO processing "$datepath
     20   echo "INFO processing "$datepath
     21   night=`echo $datepath | sed -e 's/\///g'`
     22   
     23   numganymeds=0
     24   
     25   # getting all image files for this night
     26   printprocesslog "INFO get file list for night "$datepath
     27   files=( `find $anapath/star/$datepath -type f -name '*_I.root' 2>/dev/null | sort` )
     28   if [ ${#files[@]} -eq 0 ]
     29   then
     30      printprocesslog "INFO no image files available for night "$datepath
    4931      continue
    5032   fi
    51    # process only if ganymedlogfiles is not yet there
    52    logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
    53    if ! ls $logfile >/dev/null 2>&1
    54    then
    55       # fixme: why is this double?
     33   # write data set file
     34   #dspath=$anapath/$source/datasets`printf %03d $min`min/$datepath
     35   #dspath=$anapath/$source/datasets_run/$datepath
     36   dspath=$anapath/datasets_run/$datepath
     37   makedir $dspath
     38
     39   for (( i=1 ; i<=${#files[@]} ; i++ ))
     40   do
    5641      night=`basename ${files[$i-1]} | cut -d_ -f1`
    57       runid=`basename ${files[$i-1]} | cut -d_ -f2 | cut -d. -f1`
    58       #echo "night: "$night
    59       #echo "runid: "$runid
    60       rawfilepath=$rawdata"/"$datepath"/"
    61       rawfilename=`basename ${files[$i-1]} | sed -e 's/_I.root/.fits/'`
    62       #echo $rawfilepath
    63       #echo $rawfilename
    64       rawfile=`find $rawfilepath -name $rawfilename*`
    65       printprocesslog "INFO processing "$rawfile
    66       drivefile=$auxdata"/"$datepath"/"`basename ${files[$i-1]} | cut -d_ -f1`".DRIVE_CONTROL_SOURCE_POSITION.fits"
    67       printprocesslog "INFO getting coordinates from "$drivefile
    68       tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
    69       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
    70       tstop=${tstopi}${tstopf}
    71       #echo "tstop: "$tstop
    72       if ls $drivefile >/dev/null 2>&1
     42      run=`basename ${files[$i-1]} | cut -d_ -f2`
     43      printprocesslog "DEBUG night: "$night" run: "$run
     44      dsfile=$dspath"/dataset"$night"_"$run".txt"
     45      #echo $dsfile
     46      printprocesslog "INFO writing dataset file "$dsfile
     47      # write dataset only if merpp-log is available to make sure that star-file is complete
     48      if echo $resulttable1 | grep ISDC >/dev/null 2>&1
     49      then
     50         query="SELECT LPAD(fSequenceID, 3, 0) FROM RunInfo WHERE fNight="$night" AND fRunID="$run
     51         seq=`sendquery`
     52         merpplog=`dirname ${files[$i-1]}`"/"$night"_"$seq"-merpp.log"
     53      else
     54         merpplog=`echo ${files[$i-1]} | sed -e 's/_I.root/-merpp.log/'`
     55      fi
     56      stillrunning=`find $merpplog -mmin -1 2>/dev/null`
     57      if [ -e $merpplog ] && [ "$stillrunning" == "" ]
     58      then
     59         echo `dirname ${files[$i-1]}`" "`basename ${files[$i-1]}` > $dsfile
     60      else
     61         printprocesslog "INFO merpp still running - wait with file "${files[$i-1]}
     62         continue
     63      fi
     64      # process only if ganymedlogfiles is not yet there
     65      logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
     66      if ! ls $logfile >/dev/null 2>&1
     67      then
     68         # fixme: why is this double?
     69         night=`basename ${files[$i-1]} | cut -d_ -f1`
     70         runid=`basename ${files[$i-1]} | cut -d_ -f2 | cut -d. -f1`
     71         #echo "night: "$night
     72         #echo "runid: "$runid
     73         rawfilepath=$rawdata"/"$datepath"/"
     74         rawfilename=`basename ${files[$i-1]} | sed -e 's/_I.root/.fits/'`
     75         #echo $rawfilepath
     76         #echo $rawfilename
     77         rawfile=`find $rawfilepath -name $rawfilename*`
     78         printprocesslog "INFO processing "$rawfile
     79         drivefile=$auxdata"/"$datepath"/"`basename ${files[$i-1]} | cut -d_ -f1`".DRIVE_CONTROL_SOURCE_POSITION.fits"
     80         printprocesslog "INFO getting coordinates from "$drivefile
     81         tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
     82         tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
     83         tstop=${tstopi}${tstopf}
     84         #echo "tstop: "$tstop
     85         if ls $drivefile >/dev/null 2>&1
     86            then
     87            coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop}  2>/dev/null | tail -1 2>&1` )
     88            if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
     89            then
     90               printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
     91               #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
     92               continue
     93            fi
     94            if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
     95            then
     96               printprocesslog "WARN coordinates "${coordinates[@]}
     97               #echo "WARN coordinates "${coordinates[@]}
     98               continue
     99            fi
     100         else
     101            printprocesslog "DEBUG coordinates "${coordinates[@]}
     102            #echo "DEBUG coordinates "${coordinates[@]}
     103            query="SELECT fSourceKEY FROM RunInfo WHERE fNight="$night" AND fRunID="$runid
     104            sourcekey=`sendquery`
     105            if [ "$sourcekey" == "" ]
     106            then
     107               printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
     108               #echo "WARN sourcekey empty - coordinates"${coordinates[@]}" for "$rawfile
     109               continue
     110            fi
     111            query="SELECT Round(fRightAscension,6), Round(fDeclination,6) from source WHERE fSourceKey="$sourcekey
     112            coordinates=( `sendquery` )
     113            if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
     114            then
     115               printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
     116               #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
     117               continue
     118            fi
     119            if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
     120            then
     121               printprocesslog "WARN coordinates "${coordinates[@]}
     122               #echo "WARN coordinates "${coordinates[@]}
     123               continue
     124            fi
     125         fi
     126
     127         printprocesslog "DEBUG "$logfile" "${files[$i-1]}" -"${coordinates[0]}"-"${coordinates[1]}
     128         printprocesslog "DEBUG `dirname $0`/RunGanymed.sh $dsfile "${coordinates[0]}" "${coordinates[1]}
     129         if [ "$qlasge" = "yes" ]
    73130         then
    74          coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop}  2>/dev/null | tail -1 2>&1` )
    75          if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
    76          then
    77             printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
    78             #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
    79             continue
     131            printprocesslog "$sgepath/qsub -b y -q fact_short -v AUTOMATIONSETUP=$AUTOMATIONSETUP -v dsfile=$dsfile -v ra=${coordinates[0]} -v dec=${coordinates[1]} -e bla.txt -o bla.txt `dirname $0`/RunGanymed.sh"
     132            $sgepath/qsub -b y -q fact_short -v AUTOMATIONSETUP=$AUTOMATIONSETUP -v dsfile=$dsfile -v ra=${coordinates[0]} -v dec=${coordinates[1]} -e bla.txt -o bla.txt `dirname $0`/RunGanymed.sh
     133         else
     134            `dirname $0`/RunGanymed.sh $dsfile ${coordinates[0]} ${coordinates[1]}
    80135         fi
    81          if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
    82          then
    83             printprocesslog "WARN coordinates "${coordinates[@]}
    84             #echo "WARN coordinates "${coordinates[@]}
    85             continue
    86          fi
    87       else
    88          printprocesslog "DEBUG coordinates "${coordinates[@]}
    89          #echo "DEBUG coordinates "${coordinates[@]}
    90          query="SELECT fSourceKEY FROM RunInfo WHERE fNight="$night" AND fRunID="$runid
    91          sourcekey=`sendquery`
    92          if [ "$sourcekey" == "" ]
    93          then
    94             printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
    95             #echo "WARN sourcekey empty - coordinates"${coordinates[@]}" for "$rawfile
    96             continue
    97          fi
    98          query="SELECT Round(fRightAscension,6), Round(fDeclination,6) from source WHERE fSourceKey="$sourcekey
    99          coordinates=( `sendquery` )
    100          if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
    101          then
    102             printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
    103             #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
    104             continue
    105          fi
    106          if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
    107          then
    108             printprocesslog "WARN coordinates "${coordinates[@]}
    109             #echo "WARN coordinates "${coordinates[@]}
    110             continue
    111          fi
     136         numganymeds=`echo " $numganymeds + 1 " | bc -l`
    112137      fi
    113 
    114       printprocesslog "DEBUG "$logfile" "${files[$i-1]}" -"${coordinates[0]}"-"${coordinates[1]}
    115       printprocesslog "DEBUG `dirname $0`/RunGanymed.sh $dsfile "${coordinates[0]}" "${coordinates[1]}
    116       `dirname $0`/RunGanymed.sh $dsfile ${coordinates[0]} ${coordinates[1]}
    117       numganymeds=`echo " $numganymeds + 1 " | bc -l`
    118    fi
    119 done
    120 printprocesslog "INFO found "$numganymeds" ganymeds."
    121    
    122 #   # get list of 5min-dataset-files as starting point
    123 #   printprocesslog "INFO get list of dataset files for "$min"min"
    124 #   files=( `find $dspath -type f -name 'dataset*.txt' | sort` )
    125 #   minold=5
    126 #   # loop over data set files
    127 #   # copy always 2 in one new file for double observation time
    128 #   while [ ${#files[@]} -gt 1 ]
    129 #   do
    130 #      # get double observation time and path for datasets
    131 #      min=`echo " $minold + $minold " | bc -l `
    132 #      dspathold=$dspath
    133 #      dspath=$anapath"/"$source"/datasets"`printf %03d $min`"min/"$datepath
    134 #      makedir $dspath
    135 #      # loop over already existing data set files
    136 #      for (( i=0 ; i<=${#files[@]} ; i++ ))
    137 #      do
    138 #         # number of 1st and 2nd old dataset file
    139 #         num1=`echo " $i + $i + 1" | bc -l `
    140 #         num2=`echo " $i + $i + 2 " | bc -l `
    141 #         # number of new dataset file
    142 #         dsnum=`echo " $i + 1 " | bc -l `
    143 #         # continue only if 2 dataset files are left
    144 #         if ! [ ${files[$num1]} ]
    145 #         then
    146 #            break
    147 #         fi
    148 #         ds1=$dspathold"/dataset"$num1".txt"
    149 #         ds2=$dspathold"/dataset"$num2".txt"
    150 #         dsfile=$dspath"/dataset"$dsnum".txt"
    151 #         if ! [ -e $ds1 ] || ! [ -e $ds2 ]
    152 #         then
    153 #            echo $ds1" or "$ds2" is still missing -> continue."
    154 #            continue
    155 #         fi
    156 #         # copy two old into one new dataset file
    157 #         printprocesslog "INFO writing dataset "$dsfile
    158 #         cat $ds1 > $dsfile
    159 #         cat $ds2 >> $dsfile
    160 #         # process only if ganymedlogfiles is not yet there
    161 #         logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
    162 #         if ! ls $logfile  >/dev/null 2>&1
    163 #         then
    164 #            echo $logfile
    165 #            `dirname $0`/RunGanymed.sh $source $dsfile
    166 #             numganymeds=`echo " $numganymeds + 1 " | bc -l`
    167 #         fi
    168 #      done
    169 #      # get list of new dataset files
    170 #      printprocesslog "INFO get list of dataset files for "$min"min"
    171 #      files=( `find $dspath -type f -name 'dataset*.txt' | sort` )
    172 #      minold=$min
    173 #   done
    174 
    175 #   if [ $numganymeds -gt 0 ]
    176 #   then
    177 #       if ! ps aux | grep Step2b | grep -v grep >/dev/null 2>&1
    178 #       then
    179 #         `dirname $0`/Step2b.sh &
    180 #       fi
    181 #       if ! ps aux | grep Step3 | grep -v grep >/dev/null 2>&1
    182 #       then
    183 #         `dirname $0`/Step3.sh min &
    184 #       fi
    185 #      numganymeds=0
    186 #   fi
    187 #done
     138   done
     139   printprocesslog "INFO found "$numganymeds" ganymeds."
     140done
    188141
    189142finish
     143
Note: See TracChangeset for help on using the changeset viewer.