Changeset 18723 for trunk/DataCheck/Transfer

Timestamp:

01/08/17 17:21:53 (8 years ago)

Author:

Daniela Dorner

Message:

files in the fails folder are considered now "fine" in the context of transfer

File:

• trunk/DataCheck/Transfer/CheckTransfer.sh (modified) (8 diffs)

trunk/DataCheck/Transfer/CheckTransfer.sh

@@ -101 +101 @@
    if grep "EVERYTHING" $logfile >/dev/null 2>&1
    then
-      echo " "$date" has been checked already and is fine. Please check logfile "$logfile
-      echo " "$date" has been checked already and is fine. Please check logfile "$logfile >> $logfile2 2>&1
+      if grep "fails" $logfile 2>/dev/null | grep "INFO" >/dev/null 2>&1
+      then
+         echo " "$date" has been checked already. Please check logfile "$logfile
+         echo " "$date" has been checked already. Please check logfile "$logfile >> $logfile2 2>&1
+      else
+        echo " "$date" has been checked already and is fine. Please check logfile "$logfile
+        echo " "$date" has been checked already and is fine. Please check logfile "$logfile >> $logfile2 2>&1
+      fi
       sendemail="yes"
       continue
@@ -176 +182 @@
    queryjoin="LEFT JOIN RunInfo USING(fNight,fRunID) "
    querywhere="WHERE fNight="$date2" AND NOT ISNULL(fStartTime) AND NOT ISNULL(fStopTime) AND ISNULL(fReturnCode)"
+   querywhere2="WHERE fNight="$date2" AND NOT ISNULL(fStartTime) AND NOT ISNULL(fStopTime) AND (ISNULL(fReturnCode) OR fReturnCode=0)"
    numruns=`sendquery`
    if [ "$numruns" == "" ]
@@ -192 +199 @@
    then
       numisdc=0
+   fi
+   query=$querystart"RawFileAvailISDCStatus "$queryjoin" "$querywhere2
+   numisdc2=`sendquery`
+   if [ "$numisdc2" == "" ]
+   then
+      numisdc2=0
    fi
    query=$querystart"RawFileAvailWueStatus "$queryjoin" "$querywhere
@@ -225 +238 @@
    printprocesslog "  rsynced "$numrsynced
    printprocesslog "  isdc    "$numisdc
+   printprocesslog "  isdc    "$numisdc2" (incl fails folder)"
    printprocesslog "  wue     "$numwue
    #printprocesslog "  phido   "$numphido
@@ -247 +261 @@
    echo "  rsynced "$numrsynced >> $logfile 2>&1
    echo "  isdc    "$numisdc >> $logfile 2>&1
+   echo "  isdc    "$numisdc2" (incl fails folder)" >> $logfile 2>&1
    echo "  wue     "$numwue >> $logfile 2>&1
    #echo "  phido   "$numphido >> $logfile 2>&1
@@ -500 +515 @@
          printprocesslog "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns") for "$date" [DB]"
          echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns") for "$date" [DB]" >> $logfile 2>&1
-         numpb=`echo " $numpb + 1 " | bc -l `
-         result3=$result3"1"
+         numisdctotal=`echo " $numisdc + $numisdc2 " | bc -l `
+         if ! [ $numruns -eq $numisdc2 ]
+         then
+            printprocesslog "WARN number of ingested files in archive incl fails folder ("$numisdc2") not equal to number of runs ("$numruns") for "$date" [DB]"
+            echo "WARN number of ingested files in archive incl fails folder ("$numisdc2") not equal to number of runs ("$numruns") for "$date" [DB]" >> $logfile 2>&1
+            numpb=`echo " $numpb + 1 " | bc -l `
+            result3=$result3"1"
+         else
+            numok=`echo " $numok + 1 " | bc -l `
+            result3=$result3"0"
+         fi
       else
          numok=`echo " $numok + 1 " | bc -l `
@@ -540 +564 @@
       printprocesslog "number of files does not yet agree in db ("$result3") -> do no further checking."
       echo "" >> $logfile 2>&1
-      echo "number of files does not yet agree in all sites ("$result1") -> do no further checking." >> $logfile 2>&1
+      echo "number of files does not yet agree in db ("$result3") -> do no further checking." >> $logfile 2>&1
       echo "" >> $logfile 2>&1
       # print to console
-      echo "SUMMARY for "$date
-      echo "-----------------------"
-      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "${qla[@]}" (qla) "$numdatruns" (datruns) "
+      echo "SUMMARY for "$date" (DB):" 
+      echo "-----------------------------" 
+      echo "  number of runs does not yet agree in DB: "${newdaq[0]}" (newdaq) "$numruns}" (RunInfo) "$numrsynced" (RsyncedToISDC) "$numisdc" (AvailISDC) "$numwue" (AvailWue) "
       echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile
       # print to single logfile
-      echo "SUMMARY for "$date >> $logfile 2>&1
-      echo "-----------------------" >> $logfile 2>&1
-      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "${qla[@]}" (qla) "$numdatruns" (datruns) " >> $logfile 2>&1
+      echo "SUMMARY for "$date" (DB):" >> $logfile 2>&1
+      echo "-----------------------------" >> $logfile 2>&1
+      echo "  number of runs does not yet agree in DB: "${newdaq[0]}" (newdaq) "$numruns}" (RunInfo) "$numrsynced" (RsyncedToISDC) "$numisdc" (AvailISDC) "$numwue" (AvailWue) " >> $logfile 2>&1
       echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile 2>&1
       # print to global logfile
-      echo "SUMMARY for "$date >> $logfile2 2>&1
-      echo "-----------------------" >> $logfile2 2>&1
-      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "${qla[@]}" (qla) "$numdatruns" (datruns) " >> $logfile2 2>&1
+      echo "SUMMARY for "$date" (DB):" >> $logfile2 2>&1
+      echo "-----------------------------" >> $logfile2 2>&1
+      echo "  number of runs does not yet agree in DB: "${newdaq[0]}" (newdaq) "$numruns}" (RunInfo) "$numrsynced" (RsyncedToISDC) "$numisdc" (AvailISDC) "$numwue" (AvailWue) " >> $logfile2 2>&1
       echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile2 2>&1
       continue
@@ -945 +969 @@
    if ! [ $numdatruns -eq $qla ]
    then
-      echo "WARN not all data runs are processed yet by the QLA. " 
-      echo "WARN not all data runs are processed yet by the QLA. " >> $logfile 2>&1
-      echo "WARN not all data runs are processed yet by the QLA. " >> $logfile2 2>&1
+      echo "  WARN not all data runs are processed yet by the QLA. " 
+      echo "  WARN not all data runs are processed yet by the QLA. " >> $logfile 2>&1
+      echo "  WARN not all data runs are processed yet by the QLA. " >> $logfile2 2>&1
+   fi
+   if [ ${fails[0]} -gt 0 ] || [ $numisdc -ne $numisdc2 ]
+   then
+      echo "  INFO there is "${fails[0]}" file in the fails folder. Please check. "
+      echo "  INFO there is "${fails[0]}" file in the fails folder. Please check. " >> $logfile 2>&1
+      echo "  INFO there is "${fails[0]}" file in the fails folder. Please check. " >> $logfile2 2>&1
    fi
    echo ""