Changeset 17079 for trunk/DataCheck/Transfer

Timestamp:

09/05/13 14:25:27 (11 years ago)

Author:

Daniela Dorner

Message:

added check if QLA finished callisto for all data runs

File:

• trunk/DataCheck/Transfer/CheckTransfer.sh (modified) (17 diffs)

trunk/DataCheck/Transfer/CheckTransfer.sh

@@ -85 +85 @@
 dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` )
 
-dates=( "2013/05/23" )
+#dates=( "2013/08/02" )
 #short="yes"
 
 numdaysok=0
-numdaysoklimit=5
+numdaysoklimit=10
 sumdata=0
 checklimit=5000
@@ -130 +130 @@
    localfailpath="/gpfs/fact/fact-archive/fails/raw/"$date
    wuerawpath="/fact/raw/"$date
+   qlapath="/daq/analysis/callisto/"$date
    #phidorawpath="/fhgfs/groups/app/fact-construction/raw/"$date
    
@@ -147 +148 @@
    wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
    #phido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "if [ -d $phidorawpath ]; then ls $phidorawpath/* | wc -l; du -s -b --apparent-size $phidorawpath; else echo '-1 -1 -1'; fi"` )
+   qla=( `ssh fact@161.72.93.131 "ls $qlapath/20*_C.root 2>/dev/null | wc -l"` )
    query="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM RunInfo WHERE fNight="$date2
    querystart="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM "
@@ -180 +182 @@
    #   numphido=0
    #fi
+   
+   # select number of data runs
+   query="SELECT COUNT(*) FROM RunInfo WHERE fNight="$date2
+   query=$query" AND fRunTypeKey=1"
+   numdatruns=`sendquery`
    
    printprocesslog "disk: "
@@ -190 +197 @@
    printprocesslog "  fail    "${fails[@]}
    #printprocesslog "  phido   "${phido[@]}
+   printprocesslog "  qla     "${qla[@]}
    printprocesslog "db: "
    printprocesslog "  runinfo "$numruns
@@ -196 +204 @@
    printprocesslog "  wue     "$numwue
    #printprocesslog "  phido   "$numphido
+   printprocesslog "  data    "$numdatruns
    echo "disk: " >> $logfile 2>&1
    echo "  newdaq  "${newdaq[@]} >> $logfile 2>&1
@@ -205 +214 @@
    echo "  fail    "${fails[@]} >> $logfile 2>&1
    #echo "  phido   "${phido[@]} >> $logfile 2>&1
+   echo "  qla     "${qla[@]} >> $logfile 2>&1
    echo "db: " >> $logfile 2>&1
    echo "  runinfo "$numruns >> $logfile 2>&1
@@ -211 +221 @@
    echo "  wue     "$numwue >> $logfile 2>&1
    #echo "  phido   "$numphido >> $logfile 2>&1
-   
+   echo "  datruns "$numdatruns >> $logfile 2>&1
+   
+   if ! [ $numdatruns -eq $qla ]
+   then
+      printprocesslog "WARN not all data runs are processed yet by the QLA for "$date"."
+      echo "WARN not all data runs are processed yet by the QLA for "$date >> $logfile 2>&1
+      result1="1-"
+   else
+      result1="0-"
+   fi
    # check if file are available in the different places
    if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] 
@@ -231 +250 @@
       echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date >> $logfile 2>&1
       numpb=`echo " $numpb + 1 " | bc -l `
-      result1="1"
+      result1=$result1"1"
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result1="0"
+      result1=$result1"0"
    fi
    # dl00
@@ -304 +323 @@
    #fi
    
-   if [ "$short" = "yes" ] || [ "$result1" != "0000" ]
+   if [ "$short" = "yes" ] || [ "$result1" != "0-0000" ]
    then
       printprocesslog "number of files does not yet agree in all sites ("$result1") -> do no further checking."
@@ -756 +775 @@
    printprocesslog "INFO numpb: "$numpb
    printprocesslog "result:"
-   printprocesslog "(#files-dudir-db-filesize)"
+   printprocesslog "(qla-#files-dudir-db-filesize)"
    #printprocesslog " ldawp-ldawp-ldawp-ldawp"
-   printprocesslog " ldaw-ldaw-ldaw-ldaw"
+   printprocesslog " q-ldaw-ldaw-ldaw-ldaw"
    printprocesslog " "$result1"-"$result2"-"$result3"-"$result4
    echo "INFO day ok: "$numdaysok >> $logfile 2>&1
@@ -764 +783 @@
    echo "INFO numpb: "$numpb >> $logfile 2>&1
    echo "result:" >> $logfile 2>&1
-   echo "(#files-dudir-db-filesize)" >> $logfile 2>&1
+   echo "(qla-#files-dudir-db-filesize)" >> $logfile 2>&1
    #echo " ldawp-ldawp-ldawp-ldawp" >> $logfile 2>&1
-   echo " ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1
+   echo " q-ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1
    echo " "$result1"-"$result2"-"$result3"-"$result4 >> $logfile 2>&1
    sumdatanew=`echo " ( ${daq[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l | cut -d. -f1`
@@ -776 +795 @@
    fi
 
+   echo ""
    echo "SUMMARY for "$date
    echo "-----------------------"
@@ -789 +809 @@
    #echo "arch:"${archive[0]}
    #echo "isdc:"$numisdc
-   if [ "$result1" = "0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ]
+   if [ "$result1" = "0-0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ]
    then 
       echo "  EVERYTHING is ok. "$date" can be deleted. "
@@ -797 +817 @@
       echo "    Details in the logfile "$logfile >> $logfile 2>&1
       echo "    Details in the logfile "$logfile >> $logfile2 2>&1
-   else
-      if [ "$result1" = "0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ]
+      sendemail="yes"
+   else
+      if [ "$result1" = "0-0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ]
       then 
          echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. "
@@ -809 +830 @@
          echo "    Details in the logfile "$logfile >> $logfile 2>&1
          echo "    Details in the logfile "$logfile >> $logfile2 2>&1
+         sendemail="yes"
       else
          echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile
@@ -841 +863 @@
       echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
       echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile2 2>&1
+   fi
+   if ! [ $numdatruns -eq $qla ]
+   then
+      echo "WARN not all data runs are processed yet by the QLA. " 
+      echo "WARN not all data runs are processed yet by the QLA. " >> $logfile 2>&1
+      echo "WARN not all data runs are processed yet by the QLA. " >> $logfile2 2>&1
    fi
 done