Changeset 15388 for trunk/DataCheck/QuickLook

Timestamp:

04/20/13 19:30:04 (12 years ago)

Author:

Daniela Dorner

Message:

removed sourcekey and other small improvements

File:

• trunk/DataCheck/QuickLook/Step1.sh (modified) (8 diffs)

trunk/DataCheck/QuickLook/Step1.sh

@@ -5 +5 @@
 
 # get date (before 18h there is no new data to be processed)
-datepath=`date --date="-18HOUR" +%Y/%m/%d`
-date=`date --date="-18HOUR" +%Y%m%d`
+datepath=`date --date="-19HOUR" +%Y/%m/%d`
+date=`date --date="-19HOUR" +%Y%m%d`
 #datepath="2013/04/17"
 #date="20130417"
@@ -67 +67 @@
 fi
 fileslocal=( `find $rawpath -type f | sort` )
-callistofiles=( `find $anapath -type f -name $date*-calibration.log | sort` )
+callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
 numdataruns=0
 
@@ -73 +73 @@
 makedir $rawpath
 
-echo ${#fileslocal[@]}" "${#files[@]}" "${#callistofiles[@]} 
+echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
 
 while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numdataruns -ne ${#callistofiles[@]} ]
 do
    numdataruns=0
+   echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+   printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+
    rsync_aux_file $drivefilenewdaq $drivefile
    
@@ -135 +139 @@
          if [ "$runtype" == "data" ]
          then
-            numdataruns=`echo " $numdataruns + 1 " | bc -l`
             if [ "$fileerror" = "yes" ]
             then
@@ -160 +163 @@
             echo "" >> $seqfile
 
-            tstopi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
-            tstopf=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
-            tstop=${tstopi}${tstopf}
-            coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop}  2>/dev/null | tail -1 2>&1` )
-            if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
-            then
-               printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
-               continue
-            fi
-            if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
-            then
-               printprocesslog "WARN coordinates "${coordinates[@]}
-               continue
-            fi
-            printprocesslog "DEBUG coordinates "${coordinates[@]}
-            query="SELECT fSourceKEY FROM scheduling.source WHERE "
-            query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
-            query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
-            sourcekey=`sendquery`
-            if [ "$sourcekey" == "" ]
-            then
-               printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
-            fi
+#            tstopi=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
+#            tstopf=`$factpath/fitsdump -h $localfile  2>/dev/null | grep TSTOPF | grep -E -o '[.][0-9]+'`
+#            tstop=${tstopi}${tstopf}
+#            coordinates=( `${factpath}/fitsdump ${drivefile} -c Ra_src Dec_src -r --filter='Time<'${tstop}  2>/dev/null | tail -1 2>&1` )
+#            if [ "${coordinates[0]}" == "" ] || [ "${coordinates[1]}" == "" ]
+#            then
+#               printprocesslog "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
+#               #echo "WARN couldn't get coordinates ("${coordinates[@]}") from "$drivefile
+#               continue
+#            fi
+#            if [ "${coordinates[0]}" == "0" ] || [ "${coordinates[1]}" == "0" ]
+#            then
+#               printprocesslog "WARN coordinates "${coordinates[@]}
+#               #echo "WARN coordinates "${coordinates[@]}
+#               continue
+#            fi
+#            printprocesslog "DEBUG coordinates "${coordinates[@]}
+#            query="SELECT fSourceKEY FROM scheduling.source WHERE "
+#            query=$query" fRightAscension BETWEEN "${coordinates[0]}"-0.01 AND "${coordinates[0]}"+0.01 "
+#            query=$query" AND fDeclination BETWEEN "${coordinates[1]}"-0.01 AND "${coordinates[1]}"+0.01 "
+#            sourcekey=`sendquery`
+#            if [ "$sourcekey" == "" ]
+#            then
+#               printprocesslog "WARN sourcekey empty - coordinates"${coordinates[@]}
+#            fi
+            
+            printprocesslog "INFO counting callisto logs and data files +1."
+            # get number of callisto logs
             callistocount=`ps aux | grep RunCallisto | grep -E -o '20[12][0-9][01][0-9][0-3][0-9]_[0-9][0-9][0-9]' | sort | uniq | wc -l`
+            # count data runs
+            numdataruns=`echo " $numdataruns + 1 " | bc -l`
+            #echo "numdata +1"
             
             echo "cal: "$callistocount" numdat: "$numdataruns" numcallog: "${#callistofiles[@]}
+            printprocesslog "INFO running callistos: "$callistocount" #data-runs: "$numdataruns" #callisto-logs: "${#callistofiles[@]}
             # do not overload system in case of a lot of files to be processed
             #  callistocount is set in setup.fact.lp.data
@@ -194 +206 @@
                continue
             fi
-            callistolog=`dirname $seqfile | sed -e "s/sequences/${sourcekey}\/callisto/"`"/"$night"_"$runnum"-calibration.log"
+            callistolog=`dirname $seqfile | sed -e "s/sequences/callisto/"`"/"$night"_"$runnum"-calibration.log"
             if ! [ -e $callistolog ]
             then
@@ -202 +214 @@
                rsync_aux_file $tempfilenewdaq $tempfile
                rsync_aux_file $humfilenewdaq $humfile
-               printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
-               echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
-               `dirname $0`/RunCallisto.sh $sourcekey $seqfile &
+               #printprocesslog "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
+               #echo "INFO starting RunCallisto.sh for "$sourcekey" "$seqfile
+               #`dirname $0`/RunCallisto.sh $sourcekey $seqfile &
+               printprocesslog "INFO starting RunCallisto.sh for "$seqfile
+               echo "INFO starting RunCallisto.sh for "$seqfile
+               `dirname $0`/RunCallisto.sh $seqfile &
             fi
             continue
@@ -262 +277 @@
    files=( `find $rawpathnewdaq -type f | sort` )
    fileslocal=( `find $rawpath -type f | sort` )
-   callistofiles=( `find $anapath -type f -name $date*-calibration.log | sort` )
-   
+   callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
+   echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+   printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+
+   # wait and get new file lists
    update=
    if [ ${#fileslocal[@]} -eq ${#files[@]} ] 
    then 
+      printprocesslog "INFO wait 60 seconds."
       sleep 60
       echo "sleep 60..."
+      printprocesslog "INFO get new file lists for "$datepath
       files=( `find $rawpathnewdaq -type f | sort` )
       fileslocal=( `find $rawpath -type f | sort` )
-      callistofiles=( `find $anapath -type f -name $date*-calibration.log | sort` )
+      callistofiles=( `find $anapath/callisto -type f -name $date*-calibration.log | sort` )
    fi
+   echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
+   printprocesslog "INFO status after wait end of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #data-runs:"$numdataruns
 done