Index: trunk/DataCheck/Transfer/CheckTransfer.sh
===================================================================
--- trunk/DataCheck/Transfer/CheckTransfer.sh	(revision 15599)
+++ trunk/DataCheck/Transfer/CheckTransfer.sh	(revision 16678)
@@ -7,5 +7,29 @@
 printprocesslog "INFO starting $0"
 
-source /home_nfs/isdc/fact_opr/myagent.sh
+logfile2=$logpath"/transfer/CheckTransfer.log"
+date > $logfile2 2>&1
+
+diskusage=( `ssh fact@161.72.93.131 "df -P /daq" | grep daq ` )
+# check if more than X GB are left on /daq
+if [ ${diskusage[3]} -lt 700000 ] 
+then
+   printprocesslog "WARN less than 700 GB left on /daq "
+   echo "WARN less than 700 GB left on /daq "
+   echo "WARN less than 700 GB left on /daq " >> $logfile2 2>&1
+   sendemail="yes"
+fi
+
+diskusage2=( `df -P /scratch | grep scratch ` )
+# check if more than X GB are left on /scratch
+if [ ${diskusage2[3]} -lt 500000 ] 
+then
+   printprocesslog "WARN less than 500 GB left on /scratch "
+   echo "WARN less than 500 GB left on /scratch "
+   echo "WARN less than 500 GB left on /scratch " >> $logfile2 2>&1
+   sendemail="yes"
+fi
+
+# needed for transfer to phido
+#source /home_nfs/isdc/fact_opr/myagent.sh
 
 # check first the disk in LP and on dl00
@@ -13,4 +37,5 @@
 df -h /scratch
 echo ""
+echo "" >> $logfile2 2>&1
 
 # check next the DB to know if some transfer processes failed or crashed
@@ -26,7 +51,12 @@
       #   echo $run
       #done
+      sendemail="yes"
       echo -e "\e[1;31m\x1b[5m ==>\e[00m "$1": "${runs[@]}"\e[1;31m\x1b[5m <==\e[00m "
-      echo "SELECT fNight, fRunId FROM "$1" "$where";"
+      echo "SELECT fNight, fRunId, fStartTime, fStopTime, fReturnCode FROM "$1" "$where";"
       echo "UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";"
+      echo -e " ==> "$1": "${runs[@]}" <== " >> $logfile2 2>&1
+      echo "---> Please check the DB and reset the processes if needed. " >> $logfile2 2>&1
+      echo "to check: SELECT fNight, fRunId, fStartTime, fStopTime, fReturnCode FROM "$1" "$where";" >> $logfile2 2>&1
+      echo "to reset: UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";" >> $logfile2 2>&1
    fi
 }
@@ -41,5 +71,5 @@
 # RawFileAvailISDC needs a different treatment 
 #  as return code 0 means that file is in fails folder in archive
-where="WHERE fReturnCode>0 OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) "
+where="WHERE fReturnCode>0 OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) AND fStartTime < DATE_ADD(Now(), INTERVAL -1 HOUR) "
 check_runs_in_db "RawFileAvailISDCStatus"
 
@@ -55,5 +85,5 @@
 dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` )
 
-#dates=( "2013/04/18" )
+dates=( "2013/05/23" )
 #short="yes"
 
@@ -64,8 +94,13 @@
 for date in ${dates[@]}
 do 
-   echo ""
-   echo ""
-   echo ""
-   echo $date
+   logfile=$logpath"/transfer/CheckTransfer_"`echo $date | sed -e 's/\//-/g'`".log"
+   #echo ""
+   #echo ""
+   #echo ""
+   #echo $date
+   #echo "" > $logfile 2>&1
+   #echo "" >> $logfile 2>&1
+   #echo "" >> $logfile 2>&1
+   echo $date > $logfile 2>&1
 
    # some counters
@@ -79,5 +114,5 @@
    if [ $numdaysok -ge $numdaysoklimit ] && [ $(echo " $sumdata > $checklimit " | bc -l) -eq 1 ]
    then
-      echo "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
+      printprocesslog "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
       continue
    fi
@@ -102,10 +137,11 @@
    if [ ${newdaq[0]} -eq -1 ] && [ ${daq[0]} -eq -1 ] 
    then
-      echo "INFO no data available on newdaq for "$date
+      printprocesslog "INFO no data available on newdaq for "$date
+      echo "INFO no data available on newdaq for "$date >> $logfile 2>&1
       continue
    fi
-   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
+   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* 2>/dev/null | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
    dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
-   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* | wc -l; du -L -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
+   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* 2>/dev/null | wc -l; du -L -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
    fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -L -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
    wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
@@ -144,28 +180,46 @@
    #   numphido=0
    #fi
-   echo "disk: "
-   echo "  newdaq  "${newdaq[@]}
-   echo "  daq     "${daq[@]}
-   echo "  data    "${zip[@]}
-   echo "  dl00    "${dl00[@]}
-   echo "  wue     "${wue[@]}
-   echo "  arch    "${archive[@]}
-   echo "  fail    "${fails[@]}
-   #echo "  phido   "${phido[@]}
-   echo "db: "
-   echo "  runinfo "$numruns
-   echo "  rsynced "$numrsynced
-   echo "  isdc    "$numisdc
-   echo "  wue     "$numwue
-   #echo "  phido   "$numphido
+   
+   printprocesslog "disk: "
+   printprocesslog "  newdaq  "${newdaq[@]}
+   printprocesslog "  daq     "${daq[@]}
+   printprocesslog "  data    "${zip[@]}
+   printprocesslog "  dl00    "${dl00[@]}
+   printprocesslog "  wue     "${wue[@]}
+   printprocesslog "  arch    "${archive[@]}
+   printprocesslog "  fail    "${fails[@]}
+   #printprocesslog "  phido   "${phido[@]}
+   printprocesslog "db: "
+   printprocesslog "  runinfo "$numruns
+   printprocesslog "  rsynced "$numrsynced
+   printprocesslog "  isdc    "$numisdc
+   printprocesslog "  wue     "$numwue
+   #printprocesslog "  phido   "$numphido
+   echo "disk: " >> $logfile 2>&1
+   echo "  newdaq  "${newdaq[@]} >> $logfile 2>&1
+   echo "  daq     "${daq[@]} >> $logfile 2>&1
+   echo "  data    "${zip[@]} >> $logfile 2>&1
+   echo "  dl00    "${dl00[@]} >> $logfile 2>&1
+   echo "  wue     "${wue[@]} >> $logfile 2>&1
+   echo "  arch    "${archive[@]} >> $logfile 2>&1
+   echo "  fail    "${fails[@]} >> $logfile 2>&1
+   #echo "  phido   "${phido[@]} >> $logfile 2>&1
+   echo "db: " >> $logfile 2>&1
+   echo "  runinfo "$numruns >> $logfile 2>&1
+   echo "  rsynced "$numrsynced >> $logfile 2>&1
+   echo "  isdc    "$numisdc >> $logfile 2>&1
+   echo "  wue     "$numwue >> $logfile 2>&1
+   #echo "  phido   "$numphido >> $logfile 2>&1
    
    # check if file are available in the different places
    if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] 
    then
-      echo "INFO data not available on /scratch on dl00 for "$date
+      printprocesslog "INFO data not available on /scratch on dl00 for "$date
+      echo "INFO data not available on /scratch on dl00 for "$date >> $logfile 2>&1
    fi
    if [ ${archive[0]} -eq -1 ] 
    then
-      echo "INFO data not in archive for "$date
+      printprocesslog "INFO data not in archive for "$date
+      echo "INFO data not in archive for "$date >> $logfile 2>&1
    fi
    
@@ -174,25 +228,29 @@
    if ! [ ${daq[0]} -eq -1 ]  && ! [ ${daq[0]} -eq ${newdaq[0]} ]
    then
-      echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date
+      printprocesslog "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date
+      echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date >> $logfile 2>&1
       numpb=`echo " $numpb + 1 " | bc -l `
-      result="1"
+      result1="1"
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result="0"
+      result1="0"
    fi
    # dl00
-   if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${newdaq[0]} ]
-   then
-      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+   #if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${newdaq[0]} ]
+   if ! [ ${dl00[0]} -eq ${newdaq[0]} ]
+   then
+      printprocesslog "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
       numpb=`echo " $numpb + 1 " | bc -l `
-      result=$result"1"
+      result1=$result1"1"
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result=$result"0"
+      result1=$result1"0"
    fi
    # archive
    if ! [ ${archive[0]} -eq -1 ]  && ! [ ${archive[0]} -eq ${newdaq[0]} ]
    then
-      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+      printprocesslog "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
       #check /archive/rev_1/failed
       if ! [ ${fails[0]} -eq -1 ]
@@ -201,13 +259,14 @@
          if ! [ $sum -eq ${newdaq[0]} ]
          then
-            echo "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")."
+            printprocesslog "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")."
+            echo "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")." >> $logfile 2>&1
             numpb=`echo " $numpb + 1 " | bc -l `
-            result=$result"1"
+            result1=$result1"1"
          else
             numok=`echo " $numok + 1 " | bc -l `
-            result=$result"0"
-         fi
-      else
-         result=$result"1"
+            result1=$result1"0"
+         fi
+      else
+         result1=$result1"1"
       fi
    else
@@ -215,49 +274,85 @@
       then 
          numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         result1=$result1"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result1=$result1"0"
       fi
    fi
    # wue
-   if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${newdaq[0]} ]
-   then
-      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+   #if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${newdaq[0]} ]
+   if ! [ ${wue[0]} -eq ${newdaq[0]} ]
+   then
+      printprocesslog "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
       numpb=`echo " $numpb + 1 " | bc -l `
-      result=$result"1"
+      result1=$result1"1"
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result=$result"0"
+      result1=$result1"0"
    fi
    ## phido
    #if ! [ ${phido[0]} -eq -1 ]  && ! [ ${phido[0]} -eq ${newdaq[0]} ]
    #then
-   #   echo "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+   #   printprocesslog "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
+   #   echo "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
    #   numpb=`echo " $numpb + 1 " | bc -l `
-   #   result=$result"1"
+   #   result1=$result1"1"
    #else
    #   numok=`echo " $numok + 1 " | bc -l `
-   #   result=$result"0"
+   #   result1=$result1"0"
    #fi
    
-   if [ "$short" = "yes" ] || [ "$result" != "0000" ]
-   then
-      echo $result
+   if [ "$short" = "yes" ] || [ "$result1" != "0000" ]
+   then
+      printprocesslog "number of files does not yet agree in all sites ("$result1") -> do no further checking."
+      echo "number of files does not yet agree in all sites ("$result1") -> do no further checking." >> $logfile 2>&1
+      echo "" 
+      echo "SUMMARY for "$date
+      echo "-----------------------"
+      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "
+      echo "  "$date" is not yet transfered completely. For more details, please check the logfile "$logfile
+      echo "" >> $logfile 2>&1
+      echo "SUMMARY for "$date >> $logfile 2>&1
+      echo "-----------------------" >> $logfile 2>&1
+      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) " >> $logfile 2>&1
+      echo "  "$date" is not yet transfered completely. For more details, please check the logfile "$logfile >> $logfile 2>&1
+      echo "" >> $logfile2 2>&1
+      echo "SUMMARY for "$date >> $logfile2 2>&1
+      echo "-----------------------" >> $logfile2 2>&1
+      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) " >> $logfile2 2>&1
+      echo "  "$date" is not yet transfered completely. For more details, please check the logfile "$logfile >> $logfile2 2>&1
+      #echo "number of files does not yet agree in all sites ("$result1") -> do no further checking. "
+      #echo "disk: "
+      #echo "  newdaq  "${newdaq[@]}
+      #echo "  daq     "${daq[@]}
+      #echo "  data    "${zip[@]}
+      #echo "  dl00    "${dl00[@]}
+      #echo "  wue     "${wue[@]}
+      #echo "  arch    "${archive[@]}
+      #echo "  fail    "${fails[@]}
+      ##echo "  phido   "${phido[@]}
+      #echo "db: "
+      #echo "  runinfo "$numruns
+      #echo "  rsynced "$numrsynced
+      #echo "  isdc    "$numisdc
+      #echo "  wue     "$numwue
+      ##echo "  phido   "$numphido
       continue
    fi
    
-   result=$result"-"
+   #result=$result"-"
    # check du for raw files 
    # la palma
    if ! [ ${newdaq[1]} -eq ${daq[1]} ]
    then
-      echo "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date
+      printprocesslog "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date
+      echo "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date >> $logfile 2>&1
       numdiff=`echo " $numdiff + 1 " | bc -l `
       numpb=`echo " $numpb + 1 " | bc -l `
-      result=$result"1"
+      result2="1"
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result=$result"0"
+      result2="0"
    fi
    # check du for zipped raw files 
@@ -265,70 +360,74 @@
    if ! [ ${zip[1]} -eq ${dl00[1]} ] && [ $date2 -gt 20120307 ]
    then
-      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
+      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date >> $logfile 2>&1
       if ! [ ${dl00[1]} -eq -1 ]
       then 
          numdiff=`echo " $numdiff + 1 " | bc -l `
          numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         result2=$result2"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result2=$result2"0"
       fi
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result=$result"0"
+      result2=$result2"0"
    fi
    # archive
    if ! [ ${zip[1]} -eq ${archive[1]} ]
    then
-      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
+      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date >> $logfile 2>&1
       if ! [ ${archive[1]} -eq -1 ]
       then 
          numdiff=`echo " $numdiff + 1 " | bc -l `
          numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         result2=$result2"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result2=$result2"0"
       fi
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result=$result"0"
+      result2=$result2"0"
    fi
    # wue
    if ! [ ${zip[1]} -eq ${wue[1]} ]
    then
-      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
+      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date >> $logfile 2>&1
       if ! [ ${wue[1]} -eq -1 ]
       then 
          numdiff=`echo " $numdiff + 1 " | bc -l `
          numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         result2=$result2"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result2=$result2"0"
       fi
    else
       numok=`echo " $numok + 1 " | bc -l `
-      result=$result"0"
+      result2=$result2"0"
    fi
    ## phido
    #if ! [ ${zip[1]} -eq ${phido[1]} ]
    #then
-   #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date
+   #   printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date
+   #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date >> $logfile 2>&1
    #   if ! [ ${phido[1]} -eq -1 ]
    #   then 
    #      numdiff=`echo " $numdiff + 1 " | bc -l `
    #      numpb=`echo " $numpb + 1 " | bc -l `
-   #      result=$result"1"
+   #      result2=$result2"1"
    #   else
    #      numok=`echo " $numok + 1 " | bc -l `
-   #      result=$result"0"
+   #      result2=$result2"0"
    #   fi
    #else
    #   numok=`echo " $numok + 1 " | bc -l `
-   #   result=$result"0"
+   #   result2=$result2"0"
    #fi
-   result=$result"-"
+   #result=$result"-"
 
    # check DB (only starting from 8.3.2012) (if-clause to be removed later)
@@ -338,57 +437,65 @@
       if ! [ $numruns -eq ${newdaq[0]} ]
       then
-         echo "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")"
-         numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         printprocesslog "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")"
+         echo "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")" >> $logfile 2>&1
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result3="1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result3="0"
       fi
       # dl00
       if ! [ $numruns -eq $numrsynced ]
       then
-         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
-         numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         printprocesslog "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
+         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result3=$result3"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result3=$result3"0"
       fi
       # archive
       if ! [ $numruns -eq $numisdc ]
       then
-         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
-         numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         printprocesslog "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
+         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")" >> $logfile 2>&1
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result3=$result3"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result3=$result3"0"
       fi
       # wue
       if ! [ $numruns -eq $numwue ]
       then
-         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
-         numpb=`echo " $numpb + 1 " | bc -l `
-         result=$result"1"
-      else
-         numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
+         printprocesslog "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
+         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result3=$result3"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result3=$result3"0"
       fi
       ## phido
       #if ! [ $numruns -eq $numphido ]
       #then
-      #   echo "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")"
+      #   printprocesslog "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")"
+      #   echo "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
       #   numpb=`echo " $numpb + 1 " | bc -l `
-      #   result=$result"1"
+      #   result3=$result3"1"
       #else
       #   numok=`echo " $numok + 1 " | bc -l `
-      #   result=$result"0"
+      #   result3=$result3"0"
       #fi
    fi
    
    #numdiff=0 # add for debugging so that single file sizes are not checked
-   echo "numdiff: "$numdiff
-   echo "INFO numok: "$numok
-   echo "INFO numpb: "$numpb
+   printprocesslog "numdiff: "$numdiff
+   printprocesslog "INFO numok: "$numok
+   printprocesslog "INFO numpb: "$numpb
+   echo "numdiff: "$numdiff >> $logfile 2>&1
+   echo "INFO numok: "$numok >> $logfile 2>&1
+   echo "INFO numpb: "$numpb >> $logfile 2>&1
    #if [ $numdiff -gt 0 ]
    if [ $numdiff -ge 0 ]
@@ -406,5 +513,6 @@
       daqdiffcounter=0
       daqokcounter=0
-      echo "INFO found "${#runs[@]}" rawfiles in DB."
+      printprocesslog "INFO found "${#runs[@]}" rawfiles in DB."
+      echo "INFO found "${#runs[@]}" rawfiles in DB." >> $logfile 2>&1
       for run in ${runs[@]}
       do 
@@ -432,5 +540,4 @@
             sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
          fi
-#         echo "vgl "$run": "$sizezip" - "$sizearchive
          #if ! [ ${phido[1]} -eq -1 ]
          #then
@@ -442,5 +549,6 @@
          if ! [ "$sizenewdaq" = "$sizedaq" ]
          then
-            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
+            printprocesslog "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
+            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")" >> $logfile 2>&1
             daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
          else
@@ -450,5 +558,6 @@
          if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
          then
-            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
+            printprocesslog "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
+            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")" >> $logfile 2>&1
             dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
          else
@@ -460,6 +569,7 @@
          if [ ${archive[1]} -ne -1 -a "$sizezip" != "$sizearchive" -a "$sizezip" != "$sizefails" ] 
          then
-            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
-            echo " "$sizezip"-"$sizearchive"-"${archive[1]}"-"$sizezip"-"$sizefails"-"${fails[1]}
+            printprocesslog "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
+            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")" >> $logfile 2>&1
+            #echo " "$sizezip"-"$sizearchive"-"${archive[1]}"-"$sizezip"-"$sizefails"-"${fails[1]}
             archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
          else
@@ -469,5 +579,6 @@
          if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
          then
-            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
+            printprocesslog "  "$rawfile2" data("$sizezip") wue("$sizewue")"
+            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")" >> $logfile 2>&1
             wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
          else
@@ -477,5 +588,6 @@
          #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
          #then
-         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")"
+         #   printprocesslog "  "$rawfile2" data("$sizezip") phido("$sizephido")"
+         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")" >> $logfile 2>&1
          #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
          #else
@@ -485,5 +597,6 @@
       query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
       drsruns=( `sendquery` )
-      echo "INFO found "${#drsruns[@]}" drsfiles in DB."
+      printprocesslog "INFO found "${#drsruns[@]}" drsfiles in DB."
+      echo "INFO found "${#drsruns[@]}" drsfiles in DB." >> $logfile 2>&1
       for drsrun in ${drsruns[@]}
       do 
@@ -519,5 +632,6 @@
          if ! [ "$sizenewdaq" = "$sizedaq" ]
          then
-            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
+            printprocesslog "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
+            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")" >> $logfile 2>&1
             daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
          else
@@ -527,5 +641,6 @@
          if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
          then
-            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
+            printprocesslog "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
+            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")" >> $logfile 2>&1
             dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
          else
@@ -535,5 +650,6 @@
          if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 -a "$sizearchive" != "" ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1 -a "$sizefails" != "" ]
          then
-            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
+            printprocesslog "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
+            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")" >> $logfile 2>&1
             archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
          else
@@ -543,5 +659,6 @@
          if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
          then
-            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
+            printprocesslog "  "$rawfile2" data("$sizezip") wue("$sizewue")"
+            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")" >> $logfile 2>&1
             wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
          else
@@ -551,5 +668,6 @@
          #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
          #then
-         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")"
+         #   printprocesslog "  "$rawfile2" data("$sizezip") phido("$sizephido")"
+         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")" >> $logfile 2>&1
          #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
          #else
@@ -558,12 +676,12 @@
       done
       
-      result=$result"-"
+      #result=$result"-"
       # raw files
       if [ $daqokcounter -eq ${daq[0]} ]
       then
          numok=`echo " $numok + 1 " | bc -l `
-         result=$result"0"
-      else
-         result=$result"1"
+         result4="0"
+      else
+         result4="1"
          numpb=`echo " $numpb + 1 " | bc -l `
       fi
@@ -572,8 +690,8 @@
       if [ $dl00okcounter -eq ${dl00[0]} ]
       then
-         result=$result"0"
-         numok=`echo " $numok + 1 " | bc -l `
-      else
-         result=$result"1"
+         result4=$result4"0"
+         numok=`echo " $numok + 1 " | bc -l `
+      else
+         result4=$result4"1"
          numpb=`echo " $numpb + 1 " | bc -l `
       fi
@@ -587,8 +705,8 @@
       if [ $archiveokcounter -eq ${daq[0]} ]
       then
-         result=$result"0"
-         numok=`echo " $numok + 1 " | bc -l `
-      else
-         result=$result"1"
+         result4=$result4"0"
+         numok=`echo " $numok + 1 " | bc -l `
+      else
+         result4=$result4"1"
          numpb=`echo " $numpb + 1 " | bc -l `
       fi
@@ -596,8 +714,8 @@
       if [ $wueokcounter -eq ${wue[0]} ]
       then
-         result=$result"0"
-         numok=`echo " $numok + 1 " | bc -l `
-      else
-         result=$result"1"
+         result4=$result4"0"
+         numok=`echo " $numok + 1 " | bc -l `
+      else
+         result4=$result4"1"
          numpb=`echo " $numpb + 1 " | bc -l `
       fi
@@ -605,40 +723,133 @@
       #if [ $phidookcounter -eq ${phido[0]} ]
       #then
-      #   result=$result"0"
+      #   result4=$result4"0"
       #   numok=`echo " $numok + 1 " | bc -l `
       #else
-      #   result=$result"1"
+      #   result4=$result4"1"
       #   numpb=`echo " $numpb + 1 " | bc -l `
       #fi
-      echo "INFO "$daqokcounter" files are ok on daq (raw)."
-      echo "INFO "$dl00okcounter" files are ok on dl00."
-      echo "INFO "$wueokcounter" files are ok in Wue."
-      echo "INFO "$archiveokcounter" files are ok in the archive."
-      #echo "INFO "$phidookcounter" files are ok on Phido."
-      echo "WARN "$daqdiffcounter" files have a different size on daq (raw)."
-      echo "WARN "$dl00diffcounter" files have a different size on dl00."
-      echo "WARN "$wuediffcounter" files have a different size in Wue."
-      echo "WARN "$archivediffcounter" files have a different size in the archive."
-      #echo "WARN "$phidodiffcounter" files have a different size on Phido."
+      printprocesslog "INFO "$daqokcounter" files are ok on daq (raw)."
+      printprocesslog "INFO "$dl00okcounter" files are ok on dl00."
+      printprocesslog "INFO "$wueokcounter" files are ok in Wue."
+      printprocesslog "INFO "$archiveokcounter" files are ok in the archive."
+      #printprocesslog "INFO "$phidookcounter" files are ok on Phido."
+      printprocesslog "WARN "$daqdiffcounter" files have a different size on daq (raw)."
+      printprocesslog "WARN "$dl00diffcounter" files have a different size on dl00."
+      printprocesslog "WARN "$wuediffcounter" files have a different size in Wue."
+      printprocesslog "WARN "$archivediffcounter" files have a different size in the archive."
+      #printprocesslog "WARN "$phidodiffcounter" files have a different size on Phido."
+      echo "INFO "$daqokcounter" files are ok on daq (raw)." >> $logfile 2>&1
+      echo "INFO "$dl00okcounter" files are ok on dl00." >> $logfile 2>&1
+      echo "INFO "$wueokcounter" files are ok in Wue." >> $logfile 2>&1
+      echo "INFO "$archiveokcounter" files are ok in the archive." >> $logfile 2>&1
+      #echo "INFO "$phidookcounter" files are ok on Phido." >> $logfile 2>&1
+      echo "WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
+      echo "WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
+      echo "WARN "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
+      echo "WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
+      #echo "WARN "$phidodiffcounter" files have a different size on Phido." >> $logfile 2>&1
    fi
    
    # print summary: 
-   echo "INFO day ok: "$numdaysok
-   echo "INFO numok: "$numok
-   echo "INFO numpb: "$numpb
-   echo "result:"
-   echo "(#files-dudir-db-filesize)"
-   #echo " ldawp-ldawp-ldawp-ldawp"
-   echo " ldaw-ldaw-ldaw-ldaw"
-   echo " "$result
+   printprocesslog "INFO day ok: "$numdaysok
+   printprocesslog "INFO numok: "$numok
+   printprocesslog "INFO numpb: "$numpb
+   printprocesslog "result:"
+   printprocesslog "(#files-dudir-db-filesize)"
+   #printprocesslog " ldawp-ldawp-ldawp-ldawp"
+   printprocesslog " ldaw-ldaw-ldaw-ldaw"
+   printprocesslog " "$result1"-"$result2"-"$result3"-"$result4
+   echo "INFO day ok: "$numdaysok >> $logfile 2>&1
+   echo "INFO numok: "$numok >> $logfile 2>&1
+   echo "INFO numpb: "$numpb >> $logfile 2>&1
+   echo "result:" >> $logfile 2>&1
+   echo "(#files-dudir-db-filesize)" >> $logfile 2>&1
+   #echo " ldawp-ldawp-ldawp-ldawp" >> $logfile 2>&1
+   echo " ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1
+   echo " "$result1"-"$result2"-"$result3"-"$result4 >> $logfile 2>&1
    sumdatanew=`echo " ( ${daq[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l | cut -d. -f1`
    sumdata=`echo " $sumdata + $sumdatanew " | bc -l | cut -d. -f1`
-   echo "checked alread "$sumdata" GB. "$sumdatanew" "${daq[1]}" "${zip[1]}
+   printprocesslog "checked alread "$sumdata" GB. "$sumdatanew" "${daq[1]}" "${zip[1]}
    if [ $numpb -lt 4 ]
    then 
       numdaysok=`echo " $numdaysok + 1 " | bc -l `
    fi
+
+   echo "SUMMARY for "$date
+   echo "-----------------------"
+   echo "" >> $logfile 2>&1
+   echo "SUMMARY for "$date >> $logfile 2>&1
+   echo "-----------------------" >> $logfile 2>&1
+   echo "" >> $logfile2 2>&1
+   echo "SUMMARY for "$date >> $logfile2 2>&1
+   echo "-----------------------" >> $logfile2 2>&1
+   #echo "res1:"$result1
+   #echo "res3:"$result3
+   #echo "res4:"$result4
+   #echo "arch:"${archive[0]}
+   #echo "isdc:"$numisdc
+   if [ "$result1" = "0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ]
+   then 
+      echo "  EVERYTHING is ok. "$date" can be deleted. "
+      echo "  EVERYTHING is ok. "$date" can be deleted. " >> $logfile 2>&1
+      echo "  EVERYTHING is ok. "$date" can be deleted. " >> $logfile2 2>&1
+      echo "    Details in the logfile "$logfile
+      echo "    Details in the logfile "$logfile >> $logfile 2>&1
+      echo "    Details in the logfile "$logfile >> $logfile2 2>&1
+   else
+      if [ "$result1" = "0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ]
+      then 
+         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. "
+         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " >> $logfile 2>&1
+         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " >> $logfile2 2>&1
+         echo "  TRANSFER is ok. "$date" can be deleted. "
+         echo "  TRANSFER is ok. "$date" can be deleted. " >> $logfile 2>&1
+         echo "  TRANSFER is ok. "$date" can be deleted. " >> $logfile2 2>&1
+         echo "    Details in the logfile "$logfile
+         echo "    Details in the logfile "$logfile >> $logfile 2>&1
+         echo "    Details in the logfile "$logfile >> $logfile2 2>&1
+      else
+         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile
+         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile 2>&1
+         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile2 2>&1
+         echo "    resetting jobs in the DB might be needed."
+         echo "    resetting jobs in the DB might be needed." >> $logfile 2>&1
+         echo "    resetting jobs in the DB might be needed." >> $logfile2 2>&1
+      fi
+   fi
+   if [ $daqdiffcounter -gt 0 ]
+   then 
+      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)."
+      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
+      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile2 2>&1
+   fi
+   if [ $dl00diffcounter -gt 0 ]
+   then 
+      echo "  WARN "$dl00diffcounter" files have a different size on dl00."
+      echo "  WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
+      echo "  WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile2 2>&1
+   fi
+   if [ $wuediffcounter -gt 0 ]
+   then 
+      echo "  WARN "$wuediffcounter" files have a different size in Wue."
+      echo "  WARN "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
+      echo "  WARN "$wuediffcounter" files have a different size in Wue." >> $logfile2 2>&1
+   fi
+   if [ $archivediffcounter -gt 0 ]
+   then 
+      echo "  WARN "$archivediffcounter" files have a different size in the archive."
+      echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
+      echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile2 2>&1
+   fi
 done
 
+#sendemail="yes"
+if [ "$sendemail" = "yes" ]
+then
+   echo "INFO send email with "$logfile2"to shift@fact-project.org "
+   printprocesslog "INFO send email with "$logfile2"to shift@fact-project.org "
+   cat $logfile2 | mail -s "testmail for info on deleting data" shift@fact-project.org
+fi 
+
 printprocesslog "INFO finished $0"