Changeset 16988 for trunk/DataCheck


Ignore:
Timestamp:
07/30/13 13:18:22 (11 years ago)
Author:
Daniela Dorner
Message:
completely rewritten to run only nightly ganymeds (summing more nights will be done with lightcurve macro)
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/QuickLook/Step2b.sh

    r15382 r16988  
    44printprocesslog "INFO starting $0"
    55
    6 #root=/opt/root_svn/bin/thisroot.sh
    7 #source $root
    8 #factpath=/home/fact/FACT++.in-run-fad-loss
     6# fixme: get sources from DB
     7sources=( 1 2 5 7 )
     8#sources=( 1 2 5 )
     9numganymeds=0
    910
    10 #anapath=/loc_data/analysis
    11 
    12 sources=( 1 2 5 7 )
    13 numganymeds=0
     11printprocesslog "INFO get all nights."
     12query="SELECT fNight FROM RunInfo WHERE not isnull(fSourceKey) and fNight > 20130301 GROUP BY fNight ORDER BY fNight DESC"
     13nights=( `sendquery` )
    1414
    1515for source in ${sources[@]}
    1616do
    17    # getting all nights
    18    printprocesslog "INFO get all nights for source "$source
    19    dirs=( `find $anapath/$source/datasets_run -maxdepth 3 -mindepth 3 -type d 2>/dev/null | sort` )
    20    #nights=1
    21    # write data set file
    22    dspath=$anapath/$source/datasets_night
     17   dspath=$anapath/datasets_night/$source
    2318   makedir $dspath
    24    for (( i=1 ; i<=${#dirs[@]} ; i++ ))
     19done
     20
     21for night in ${nights[@]}
     22do
     23   echo $night
     24   for source in ${sources[@]}
    2525   do
    26       numdsfiles=`find ${dirs[$i-1]} -type f | wc -l`
    27       imgpath=`echo ${dirs[$i-1]} | sed -e 's/datasets_run/star/'`
    28       numimgfiles=`find $imgpath -type f -name '*_I.root' | wc -l`
    29       echo ${dirs[$i-1]}" "$numdsfiles" "$numimgfiles
    30       night=`echo ${dirs[$i-1]} | cut -d/ -f6-8 | sed -e 's/\///g'`
    31       echo $night
     26      dspath=$anapath/datasets_night/$source
    3227      dsfile=$dspath"/dataset"$night".txt"
    33       numold=`cat $dsfile | wc -l`
    34       echo ${dirs[$i-1]}" "$numdsfiles" "$numimgfiles" "$numold
    35       echo $dsfile
    36       printprocesslog "INFO writing dataset file "$dsfile
    37       echo "INFO writing dataset file "$dsfile
    38       #cat ${dirs[$i-1]}/dataset*.txt
    39       cat ${dirs[$i-1]}/dataset*.txt > $dsfile
    40       # process only if ganymedlogfiles is not yet there
    41       logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
    42       echo $logfile
    43       # this does not work as there might be dataset files already pr
    44       #if [ $numdsfiles -ne $numimgfiles ]
    45       if [ $numold -lt $numimgfiles ]
     28      if ! ls $dsfile >/dev/null 2>&1
    4629      then
    47          printprocesslog "INFO remove $logfiles (new files added to dataset) "
    48          rm $logfile
     30         echo "# source "$source" for night "$night > $dsfile
     31         linesold=0
     32      else
     33         lines=( `wc -l $dsfile` )
     34         linesold[$source]=${lines[0]}
    4935      fi
    50       if ! ls $logfile >/dev/null 2>&1
     36   done
     37
     38   inpath=$anapath/star/`echo $night | cut -c 1-4`/`echo $night | cut -c 5-6`/`echo $night | cut -c 7-8`
     39   #files=`find $inpath -name '*_I.root'`
     40   files=( `ls $inpath/*_I.root 2>/dev/null` )
     41   if [ ${#files[@]} -eq 0 ]
     42   then
     43      continue
     44   fi
     45   echo "files: "${#files[@]}
     46   for file in ${files[@]}
     47   do
     48      run=`basename $file | cut -d_ -f2`
     49      query="SELECT fSourceKey FROM RunInfo WHERE fNight="$night" AND fRunId="$run" GROUP BY fSourceKey"
     50      source=`sendquery`
     51      if [ $source -eq 0 ]
    5152      then
    52          echo "`dirname $0`/RunGanymed.sh $source $dsfile"
    53          `dirname $0`/RunGanymed.sh $source $dsfile
    54          numganymeds=`echo " $numganymeds + 1 " | bc -l`
     53         continue
    5554      fi
    56    done
     55      dspath=$anapath/datasets_night/$source
     56      dsfile=$dspath"/dataset"$night".txt"
     57      if ! grep `basename $file` $dsfile >/dev/null 2>&1
     58      then
     59         if echo $resulttable1 | grep ISDC >/dev/null 2>&1
     60         then
     61            query="SELECT LPAD(fSequenceID, 3, 0) FROM RunInfo WHERE fNight="$night" AND fRunID="$run
     62            seq=`sendquery`
     63            merpplog=`dirname $file`"/"$night"_"$seq"-merpp.log"
     64         else
     65            merpplog=`echo $file | sed -e 's/_I.root/-merpp.log/'`
     66         fi
     67         echo $merpplog
     68         stillrunning=`find $merpplog -mmin -1 2>/dev/null`
     69         if [ -e $merpplog ] && [ "$stillrunning" == "" ]
     70         then
     71            echo `dirname $file`" "`basename $file` >> $dsfile
     72            echo `dirname $file`" "`basename $file`
     73         else
     74            printprocesslog "INFO merpp still running - wait with file "$file
     75            continue
     76         fi
     77      fi
     78   done
    5779   
    58 #   # get list of dataset files for 1 night as starting point
    59 #   printprocesslog "INFO get list of dataset files for "$nights" nights"
    60 #   files=( `find $dspath -type f -name 'dataset*.txt' | sort` )
    61 #   nightsold=1
    62 #   # loop over data set files
    63 #   # copy always 2 in one new file for double observation time
    64 #   while [ ${#files[@]} -gt 1 ]
    65 #   do
    66 #      # get double observation time and path for datasets
    67 #      nights=`echo " $nightsold + $nightsold " | bc -l `
    68 #      dspathold=$dspath
    69 #      dspath=$anapath"/"$source"/datasets"`printf %04d $nights`"nights"
    70 #      makedir $dspath
    71 #      # loop over already existing data set files
    72 #      for (( i=0 ; i<=${#files[@]} ; i++ ))
    73 #      do
    74 #         # number of 1st and 2nd old dataset file
    75 #         num1=`echo " $i + $i + 1" | bc -l `
    76 #         num2=`echo " $i + $i + 2 " | bc -l `
    77 #         # number of new dataset file
    78 #         dsnum=`echo " $i + 1 " | bc -l `
    79 #         # continue only if 2 dataset files are left
    80 #         if ! [ ${files[$num1]} ]
    81 #         then
    82 #            break
    83 #         fi
    84 #         ds1=$dspathold"/dataset"$num1".txt"
    85 #         ds2=$dspathold"/dataset"$num2".txt"
    86 #         dsfile=$dspath"/dataset"$dsnum".txt"
    87 #         # copy two old into one new dataset file
    88 #         printprocesslog "INFO writing dataset "$dsfile
    89 #         echo "INFO writing dataset "$dsfile
    90 #         cat $ds1 > $dsfile
    91 #         cat $ds2 >> $dsfile
    92 #         # process only if ganymedlogfiles is not yet there
    93 #         logfile=`echo $dsfile | sed -e 's/datasets/ganymed/' -e 's/dataset//' -e 's/[.]txt/-ganymed[.]log/'`
    94 #         if ! ls $logfile  >/dev/null 2>&1
    95 #         then
    96 #            `dirname $0`/RunGanymed.sh $source $dsfile
    97 #            numganymeds=`echo " $numganymeds + 1 " | bc -l`
    98 #         fi
    99 #      done
    100 #      # get list of new dataset files
    101 #      printprocesslog "INFO get list of dataset files for "$nights" nights"
    102 #      files=( `find $dspath -type f -name 'dataset*.txt' | sort` )
    103 #      nightsold=$nights
    104 #   done
    105    
    106     echo "found "$numganymeds" ganymeds."
    107    #if [ $numganymeds -gt 0 ]
    108    #then
    109    #    if ! ps aux | grep Step3 | grep -v grep >/dev/null 2>&1
    110    #    then
    111    #      `dirname $0`/Step3.sh nights &
    112    #    fi
    113    #   numganymeds=0
    114    #fi
    115    
     80   for source in ${sources[@]}
     81   do
     82      dspath=$anapath/datasets_night/$source
     83      dsfile=$dspath"/dataset"$night".txt"
     84      if ls $dsfile >/dev/null 2>&1
     85      then
     86         lines=( `wc -l $dsfile` )
     87         linesnew[$source]=${lines[0]}
     88         if [ ${lines[0]} -le 1 ]
     89         then
     90            # not ideal - what if number of files stays the same, but files change?
     91            rm -v $dsfile
     92            continue
     93         fi
     94         echo "new: "${linesnew[$source]}
     95         echo "old: "${linesold[$source]}
     96         if [ ${linesnew[$source]} -le ${linesold[$source]} ]
     97         then
     98            echo "nothing new"
     99            continue
     100         else
     101            ls $dsfile
     102         fi
     103      fi
     104      query="SELECT Round(fRightAscension,6), Round(fDeclination,6) from source WHERE fSourceKey="$source
     105      coordinates=( `sendquery` )
     106      if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
     107      then
     108         printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
     109         #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
     110         continue
     111      fi
     112      if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
     113      then
     114         printprocesslog "WARN coordinates "${coordinates[@]}
     115         #echo "WARN coordinates "${coordinates[@]}
     116         continue
     117      fi
     118      if ls $dsfile >/dev/null 2>&1
     119      then
     120         outpath=$anapath/ganymeds_night/$source
     121         logfile=$outpath"/ganymed"$night".log"
     122         echo $logfile
     123         #ls $logfile
     124         if ! ls $logfile >/dev/null 2>&1
     125         then
     126            ls $dsfile
     127            if [ "$qlasge" = "yes" ]
     128            then
     129               printprocesslog "$sgepath/qsub -b y -q fact_short -v AUTOMATIONSETUP=$AUTOMATIONSETUP -v dsfile=$dsfile -v ra=${coordinates[0]} -v dec=${coordinates[1]} -e bla.txt -o bla.txt `dirname $0`/RunGanymed.sh"
     130               $sgepath/qsub -b y -q fact_short -v AUTOMATIONSETUP=$AUTOMATIONSETUP -v dsfile=$dsfile -v ra=${coordinates[0]} -v dec=${coordinates[1]} -e bla.txt -o bla.txt `dirname $0`/RunGanymed.sh
     131            else
     132               `dirname $0`/RunGanymed.sh $dsfile ${coordinates[0]} ${coordinates[1]}
     133            fi
     134            #echo "`dirname $0`/RunGanymed.sh $source $dsfile"
     135            #`dirname $0`/RunGanymed.sh $source $dsfile
     136            numganymeds=`echo " $numganymeds + 1 " | bc -l`
     137         fi
     138      fi
     139   done
     140
     141   echo "found "$numganymeds" ganymeds."
    116142done
    117143
Note: See TracChangeset for help on using the changeset viewer.