#!/bin/bash
#
# This script checks whether data can be deleted
#

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"

numchecktransfer=`/usr/sbin/lsof $0 | grep -o -c $0`
if [ $numchecktransfer -gt 1 ]
then
   printprocesslog "INFO "$0" already running -> exit. "
   echo "INFO "$0" already running -> exit. "
   finish
fi

logfile2=$logpath"/transfer/CheckTransfer.log"
date > $logfile2 2>&1

diskusage=( `ssh fact@161.72.93.131 "df -P /daq" | grep daq ` )
# check if more than X GB are left on /daq
if [ ${diskusage[3]} -lt 700000 ] 
then
   printprocesslog "DISK less than 700 GB left on /daq ("${diskusage[3]}")"
   echo "WARN less than 700 GB left on /daq ("${diskusage[3]}")"
   echo "WARN less than 700 GB left on /daq ("${diskusage[3]}")" >> $logfile2 2>&1
   sendemail="yes"
fi

diskusage2=( `df -P /scratch | grep scratch ` )
# check if more than X GB are left on /scratch
if [ ${diskusage2[3]} -lt 500000 ] 
then
   printprocesslog "DISK less than 500 GB left on /scratch ("${diskusage2[3]}")"
   echo "WARN less than 500 GB left on /scratch "${diskusage2[3]}")"
   echo "WARN less than 500 GB left on /scratch "${diskusage2[3]}")" >> $logfile2 2>&1
   sendemail="yes"
fi

# needed for transfer to phido
#source /home_nfs/isdc/fact_opr/myagent.sh

# check first the disk in LP and on dl00
ssh fact@161.72.93.131 "df -h /*da*"
df -h /scratch
echo ""
echo "" >> $logfile2 2>&1

# check next the DB to know if some transfer processes failed or crashed
function check_runs_in_db()
{
   query="SELECT "$toquery" FROM "$1" "$where
   #echo $query
   runs=( `sendquery $query` )
   if [ ${#runs[@]} -gt 0 ]
   then
      #for run in ${runs[@]}
      #do
      #   echo $run
      #done
      sendemail="yes"
      echo -e "\e[1;31m\x1b[5m ==>\e[00m "$1": "${runs[@]}"\e[1;31m\x1b[5m <==\e[00m "
      echo "SELECT fNight, fRunId, fStartTime, fStopTime, fReturnCode FROM "$1" "$where";"
      echo "UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";"
      echo -e " ==> "$1": "${runs[@]}" <== " >> $logfile2 2>&1
      echo "---> Please check the DB and reset the processes if needed. " >> $logfile2 2>&1
      echo "to check: SELECT fNight, fRunId, fStartTime, fStopTime, fReturnCode FROM "$1" "$where";" >> $logfile2 2>&1
      echo "to reset: UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";" >> $logfile2 2>&1
   fi
}
# get information of runs where transfer had a problem
toquery="fNight, fRunID, fStartTime, fStopTime, fAvailable, fProcessingSiteKey, fReturnCode "
toquery="CONCAT(fNight, '_', fRunID, '(', fStartTime, '-', fStopTime, ':', fReturnCode, ')') "
toquery="CONCAT(fNight, '_', fRunID, ':', fReturnCode) "
toquery="IF (ISNULL(fReturnCode), CONCAT(fNight, '_', fRunID, 'crashed'), CONCAT(fNight, '_', fRunID, 'failed', fReturnCode)) "
where="WHERE NOT ISNULL(fReturnCode) OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) AND fStartTime < DATE_ADD(Now(), INTERVAL -5 HOUR) "
check_runs_in_db "RawFileRsyncedISDCStatus"
check_runs_in_db "RawFileAvailWueStatus"
# RawFileAvailISDC needs a different treatment 
#  as return code 0 means that file is in fails folder in archive
where="WHERE fReturnCode>0 OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) AND fStartTime < DATE_ADD(Now(), INTERVAL -1 HOUR) "
check_runs_in_db "RawFileAvailISDCStatus"

# get nights from directory in LP
dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` )

numdaysok=0
numdaysoklimit=10
sumdata=0
checklimit=5000
for date in ${dates[@]}
do 
   logfile=$logpath"/transfer/CheckTransfer_"`echo $date | sed -e 's/\//-/g'`".log"

   echo ""
   echo "" >> $logfile2 2>&1
   echo "Processing "$date" ..." 
   echo "Processing "$date" ..." >> $logfile2 2>&1

   # avoid that already checked days are checked again
   if grep "EVERYTHING" $logfile >/dev/null 2>&1
   then
      echo " "$date" has been checked already and is fine. Please check logfile "$logfile
      echo " "$date" has been checked already and is fine. Please check logfile "$logfile >> $logfile2 2>&1
      sendemail="yes"
      continue
   fi
   if [ "$certaindate" != "" ]
   then
      checkstring=`echo $certaindate | grep -E -o '^20[0-9][0-9]\/[01][0-9]\/[0-3][0-9]$'`
      if [ "$checkstring" = "" ]
      then
         echo "Please give the variable certaindate in the correct format (YYYY/MM/DD)"
         finish
      fi
      if [ "$certaindate" != "$date" ]
      then
         printprocesslog "INFO continue, as certaindate has been set to "$certaindate
         echo " continue, as certaindate has been set to "$certaindate
         continue
      fi
   fi

   # some counters
   numdiff=0
   numok=0
   numpb=0
   
   # check always only $numdaysoklimit days
   # and require at least $checklimit GB that have been checked 
   #  remark: bc: expr1 < expr2: the result is 1 if expr1 is strictly less than expr2
   if [ $numdaysok -ge $numdaysoklimit ] && [ $(echo " $sumdata > $checklimit " | bc -l) -eq 1 ]
   then
      printprocesslog "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
      continue
   fi
   
   # get paths
   date2=`echo $date | sed -e 's/\///g'`
#   lprawpath="/daq/raw/"$date
   lprawpath="/newdaq/raw/"$date
#   lprawpath2="/loc_data/raw/"$date
   lprawpath2="/daq/raw/"$date
   lpziprawpath="/loc_data/zipraw/"$date
   localrawpath="/scratch/from_lapalma/raw/"$date
   localrawpath3="/fact/raw/"$date
#   localfailpath="/archive/fact/fails/raw/"$date
   localfailpath="/gpfs/fact/fact-archive/fails/raw/"$date
   wuerawpath="/fact/raw/"$date
   qlapath="/daq/analysis/callisto/"$date
   #phidorawpath="/fhgfs/groups/app/fact-construction/raw/"$date
   
   # get disk usage and number of files for directory
   #   on newdaq files with more than 3 digits as run number are excluded
   newdaq=( `ssh fact@161.72.93.131 "if [ -d $lprawpath ]; then ls $lprawpath/${date2}_[0-9][0-9][0-9][.]* | wc -l; du -s -b --apparent-size $lprawpath; else echo '-1 -1 -1'; fi"` )
   daq=( `ssh fact@161.72.93.131 "if [ -d $lprawpath2 ]; then ls $lprawpath2/* | wc -l; du -s -b --apparent-size $lprawpath2; else echo '-1 -1 -1'; fi"` )
   if [ ${newdaq[0]} -eq -1 ] && [ ${daq[0]} -eq -1 ] 
   then
      printprocesslog "INFO no data available on newdaq for "$date
      echo "INFO no data available on newdaq for "$date >> $logfile 2>&1
      continue
   fi
   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* 2>/dev/null | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
   dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* 2>/dev/null | wc -l; du -L -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
   fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -L -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
   wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
   #phido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "if [ -d $phidorawpath ]; then ls $phidorawpath/* | wc -l; du -s -b --apparent-size $phidorawpath; else echo '-1 -1 -1'; fi"` )
   qla=( `ssh fact@161.72.93.131 "ls $qlapath/20*_C.root 2>/dev/null | wc -l"` )
   query="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM RunInfo WHERE fNight="$date2
   querystart="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM "
   queryjoin="LEFT JOIN RunInfo USING(fNight,fRunID) "
   querywhere="WHERE fNight="$date2" AND NOT ISNULL(fStartTime) AND NOT ISNULL(fStopTime) AND ISNULL(fReturnCode)"
   numruns=`sendquery`
   if [ "$numruns" == "" ]
   then
      numruns=0
   fi
   query=$querystart"RawFileRsyncedISDCStatus "$queryjoin" "$querywhere
   numrsynced=`sendquery`
   if [ "$numrsynced" == "" ]
   then
      numrsynced=0
   fi
   query=$querystart"RawFileAvailISDCStatus "$queryjoin" "$querywhere
   numisdc=`sendquery`
   if [ "$numisdc" == "" ]
   then
      numisdc=0
   fi
   query=$querystart"RawFileAvailWueStatus "$queryjoin" "$querywhere
   numwue=`sendquery`
   if [ "$numwue" == "" ]
   then
      numwue=0
   fi
   #query=$querystart"RawFileAvailPhidoStatus "$queryjoin" "$querywhere
   #numphido=`sendquery`
   #if [ "$numphido" == "" ]
   #then
   #   numphido=0
   #fi
   
   # select number of data runs
   query="SELECT COUNT(*) FROM RunInfo WHERE fNight="$date2
   query=$query" AND fRunTypeKey=1"
   numdatruns=`sendquery`
   
   printprocesslog "disk: "
   printprocesslog "  newdaq  "${newdaq[@]}
   printprocesslog "  daq     "${daq[@]}
   printprocesslog "  data    "${zip[@]}
   printprocesslog "  dl00    "${dl00[@]}
   printprocesslog "  wue     "${wue[@]}
   printprocesslog "  arch    "${archive[@]}
   printprocesslog "  fail    "${fails[@]}
   #printprocesslog "  phido   "${phido[@]}
   printprocesslog "  qla     "${qla[@]}
   printprocesslog "db: "
   printprocesslog "  runinfo "$numruns
   printprocesslog "  rsynced "$numrsynced
   printprocesslog "  isdc    "$numisdc
   printprocesslog "  wue     "$numwue
   #printprocesslog "  phido   "$numphido
   printprocesslog "  data    "$numdatruns
   echo "disk: " >> $logfile 2>&1
   echo "  newdaq  "${newdaq[@]} >> $logfile 2>&1
   echo "  daq     "${daq[@]} >> $logfile 2>&1
   echo "  data    "${zip[@]} >> $logfile 2>&1
   echo "  dl00    "${dl00[@]} >> $logfile 2>&1
   echo "  wue     "${wue[@]} >> $logfile 2>&1
   echo "  arch    "${archive[@]} >> $logfile 2>&1
   echo "  fail    "${fails[@]} >> $logfile 2>&1
   #echo "  phido   "${phido[@]} >> $logfile 2>&1
   echo "  qla     "${qla[@]} >> $logfile 2>&1
   echo "db: " >> $logfile 2>&1
   echo "  runinfo "$numruns >> $logfile 2>&1
   echo "  rsynced "$numrsynced >> $logfile 2>&1
   echo "  isdc    "$numisdc >> $logfile 2>&1
   echo "  wue     "$numwue >> $logfile 2>&1
   #echo "  phido   "$numphido >> $logfile 2>&1
   echo "  datruns "$numdatruns >> $logfile 2>&1
   
   if ! [ $numdatruns -eq $qla ]
   then
      printprocesslog "WARN not all data runs are processed yet by the QLA for "$date"."
      echo "WARN not all data runs are processed yet by the QLA for "$date >> $logfile 2>&1
      result1="1-"
   else
      result1="0-"
   fi
   # check if file are available in the different places
   if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] 
   then
      printprocesslog "INFO data not available on /scratch on dl00 for "$date
      echo "INFO data not available on /scratch on dl00 for "$date >> $logfile 2>&1
   fi
   if [ ${archive[0]} -eq -1 ] 
   then
      printprocesslog "INFO data not in archive for "$date
      echo "INFO data not in archive for "$date >> $logfile 2>&1
   fi
   
   # check if number of files agree in the different places
   # lp
   if ! [ ${daq[0]} -eq -1 ]  && ! [ ${daq[0]} -eq ${newdaq[0]} ]
   then
      printprocesslog "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date
      echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date >> $logfile 2>&1
      numpb=`echo " $numpb + 1 " | bc -l `
      result1=$result1"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result1=$result1"0"
   fi
   # dl00
   #if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${newdaq[0]} ]
   if ! [ ${dl00[0]} -eq ${newdaq[0]} ]
   then
      printprocesslog "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
      numpb=`echo " $numpb + 1 " | bc -l `
      result1=$result1"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result1=$result1"0"
   fi
   # archive
   if ! [ ${archive[0]} -eq -1 ]  && ! [ ${archive[0]} -eq ${newdaq[0]} ]
   then
      printprocesslog "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
      #check /archive/rev_1/failed
      if ! [ ${fails[0]} -eq -1 ]
      then
         sum=`echo " ${fails[0]} + ${archive[0]} " | bc -l `
         if ! [ $sum -eq ${newdaq[0]} ]
         then
            printprocesslog "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")."
            echo "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")." >> $logfile 2>&1
            numpb=`echo " $numpb + 1 " | bc -l `
            result1=$result1"1"
         else
            numok=`echo " $numok + 1 " | bc -l `
            result1=$result1"0"
         fi
      else
         result1=$result1"1"
      fi
   else
      if [ ${archive[0]} -eq -1 ]
      then 
         numpb=`echo " $numpb + 1 " | bc -l `
         result1=$result1"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result1=$result1"0"
      fi
   fi
   # wue
   #if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${newdaq[0]} ]
   if ! [ ${wue[0]} -eq ${newdaq[0]} ]
   then
      printprocesslog "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
      numpb=`echo " $numpb + 1 " | bc -l `
      result1=$result1"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result1=$result1"0"
   fi
   ## phido
   #if ! [ ${phido[0]} -eq -1 ]  && ! [ ${phido[0]} -eq ${newdaq[0]} ]
   #then
   #   printprocesslog "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
   #   echo "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
   #   numpb=`echo " $numpb + 1 " | bc -l `
   #   result1=$result1"1"
   #else
   #   numok=`echo " $numok + 1 " | bc -l `
   #   result1=$result1"0"
   #fi
   
   short2=
   if [ "$result1" != "0-0000" ] && [ "$short" != "no" ]
   then
      short2="yes"
   fi
   
   if [ "$short2" = "yes" ] 
   then
      printprocesslog "number of files does not yet agree in all sites ("$result1") -> do no further checking."
      echo "" >> $logfile 2>&1
      echo "number of files does not yet agree in all sites ("$result1") -> do no further checking." >> $logfile 2>&1
      echo "" >> $logfile 2>&1
      # print to console
      echo "SUMMARY for "$date
      echo "-----------------------"
      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "${qla[@]}" (qla) "$numdatruns" (datruns) "
      echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile
      # print to single logfile
      echo "SUMMARY for "$date >> $logfile 2>&1
      echo "-----------------------" >> $logfile 2>&1
      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "${qla[@]}" (qla) "$numdatruns" (datruns) " >> $logfile 2>&1
      echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile 2>&1
      # print to global logfile
      echo "SUMMARY for "$date >> $logfile2 2>&1
      echo "-----------------------" >> $logfile2 2>&1
      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "${qla[@]}" (qla) "$numdatruns" (datruns) " >> $logfile2 2>&1
      echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile2 2>&1
      continue
   fi
   
   # check du for raw files 
   # la palma
   if ! [ ${newdaq[1]} -eq ${daq[1]} ]
   then
      printprocesslog "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date
      echo "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date >> $logfile 2>&1
      numdiff=`echo " $numdiff + 1 " | bc -l `
      numpb=`echo " $numpb + 1 " | bc -l `
      result2="1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result2="0"
   fi
   # check du for zipped raw files 
   # dl00
   if ! [ ${zip[1]} -eq ${dl00[1]} ] && [ $date2 -gt 20120307 ]
   then
      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date >> $logfile 2>&1
      if ! [ ${dl00[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result2=$result2"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result2=$result2"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result2=$result2"0"
   fi
   # archive
   if ! [ ${zip[1]} -eq ${archive[1]} ]
   then
      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date >> $logfile 2>&1
      if ! [ ${archive[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result2=$result2"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result2=$result2"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result2=$result2"0"
   fi
   # wue
   if ! [ ${zip[1]} -eq ${wue[1]} ]
   then
      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date >> $logfile 2>&1
      if ! [ ${wue[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result2=$result2"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result2=$result2"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result2=$result2"0"
   fi
   ## phido
   #if ! [ ${zip[1]} -eq ${phido[1]} ]
   #then
   #   printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date
   #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date >> $logfile 2>&1
   #   if ! [ ${phido[1]} -eq -1 ]
   #   then 
   #      numdiff=`echo " $numdiff + 1 " | bc -l `
   #      numpb=`echo " $numpb + 1 " | bc -l `
   #      result2=$result2"1"
   #   else
   #      numok=`echo " $numok + 1 " | bc -l `
   #      result2=$result2"0"
   #   fi
   #else
   #   numok=`echo " $numok + 1 " | bc -l `
   #   result2=$result2"0"
   #fi
   #result=$result"-"

   # check DB (only starting from 8.3.2012) (if-clause to be removed later)
   if [ $date2 -gt 20120307 ]
   then
      # lp
      if ! [ $numruns -eq ${newdaq[0]} ]
      then
         printprocesslog "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")"
         echo "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")" >> $logfile 2>&1
         numpb=`echo " $numpb + 1 " | bc -l `
         result3="1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result3="0"
      fi
      # dl00
      if ! [ $numruns -eq $numrsynced ]
      then
         printprocesslog "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
         numpb=`echo " $numpb + 1 " | bc -l `
         result3=$result3"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result3=$result3"0"
      fi
      # archive
      if ! [ $numruns -eq $numisdc ]
      then
         printprocesslog "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")" >> $logfile 2>&1
         numpb=`echo " $numpb + 1 " | bc -l `
         result3=$result3"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result3=$result3"0"
      fi
      # wue
      if ! [ $numruns -eq $numwue ]
      then
         printprocesslog "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
         numpb=`echo " $numpb + 1 " | bc -l `
         result3=$result3"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result3=$result3"0"
      fi
      ## phido
      #if ! [ $numruns -eq $numphido ]
      #then
      #   printprocesslog "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")"
      #   echo "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
      #   numpb=`echo " $numpb + 1 " | bc -l `
      #   result3=$result3"1"
      #else
      #   numok=`echo " $numok + 1 " | bc -l `
      #   result3=$result3"0"
      #fi
   fi
   
   #numdiff=0 # add for debugging so that single file sizes are not checked
   printprocesslog "numdiff: "$numdiff
   printprocesslog "INFO numok: "$numok
   printprocesslog "INFO numpb: "$numpb
   echo "numdiff: "$numdiff >> $logfile 2>&1
   echo "INFO numok: "$numok >> $logfile 2>&1
   echo "INFO numpb: "$numpb >> $logfile 2>&1
   #if [ $numdiff -gt 0 ]
   if [ $numdiff -ge 0 ]
   then
      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2
      runs=( `sendquery` )
      archivediffcounter=0
      archiveokcounter=0
      wuediffcounter=0
      wueokcounter=0
      #phidodiffcounter=0
      #phidookcounter=0
      dl00diffcounter=0
      dl00okcounter=0
      daqdiffcounter=0
      daqokcounter=0
      printprocesslog "INFO found "${#runs[@]}" rawfiles in DB."
      echo "INFO found "${#runs[@]}" rawfiles in DB." >> $logfile 2>&1
      for run in ${runs[@]}
      do 
         rawfile=$date2"_"`printf %03d $run`".fits"
         #rawfile2=$rawfile".gz"
         rawfile2=$rawfile".*z"
         printprocesslog "INFO checking "$rawfile
         
         # get file sizes for run
         #sizenewdaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizenewdaq=( `ssh fact@161.72.93.131 "ls -lH ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizedaq=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         if ! [ ${dl00[1]} -eq -1 ]
         then
            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${archive[1]} -eq -1 ]
         then
            sizearchive=( `ls -lH $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${fails[1]} -eq -1 ]
         then
            sizefails=( `ls -lH $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${wue[1]} -eq -1 ]
         then
            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         fi
         #if ! [ ${phido[1]} -eq -1 ]
         #then
         #   sizephido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "ls -l $phidorawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         #fi
         
         # check file sizes for run
         # lp
         if ! [ "$sizenewdaq" = "$sizedaq" ]
         then
            printprocesslog "WARN "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")" >> $logfile 2>&1
            daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
         else
            daqokcounter=`echo " $daqokcounter + 1 " | bc -l `
         fi
         # dl00
         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
         then
            printprocesslog "WARN "$rawfile2" data("$sizezip") dl00("$sizedl00")"
            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")" >> $logfile 2>&1
            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
         else
            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
         fi
         # archive
         #if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 -a "$sizearchive" != "" ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1 -a "$sizefails" != "" ] #not yet ingested files are treated wrongly
         #if [ ${archive[1]} -ne -1 ] && [ "$sizezip" != "$sizearchive" -o "$sizezip" != "$sizefails" ] 
         if [ ${archive[1]} -ne -1 -a "$sizezip" != "$sizearchive" -a "$sizezip" != "$sizefails" ] 
         then
            printprocesslog "WARN "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")" >> $logfile 2>&1
            #echo " "$sizezip"-"$sizearchive"-"${archive[1]}"-"$sizezip"-"$sizefails"-"${fails[1]}
            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
         else
            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
         fi
         # wue
         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
         then
            printprocesslog "WARN "$rawfile2" data("$sizezip") wue("$sizewue")"
            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")" >> $logfile 2>&1
            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
         else
            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
         fi
         ## phido
         #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
         #then
         #   printprocesslog "  "$rawfile2" data("$sizezip") phido("$sizephido")"
         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")" >> $logfile 2>&1
         #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
         #else
         #   phidookcounter=`echo " $phidookcounter + 1 " | bc -l `
         #fi
      done
      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
      drsruns=( `sendquery` )
      printprocesslog "INFO found "${#drsruns[@]}" drsfiles in DB."
      echo "INFO found "${#drsruns[@]}" drsfiles in DB." >> $logfile 2>&1
      for drsrun in ${drsruns[@]}
      do 
         rawfile=$date2"_"`printf %03d $drsrun`".drs.fits"
         #rawfile2=$rawfile".gz"
         rawfile2=$rawfile".*z"
         # get file sizes for run
         #sizenewdaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizenewdaq=( `ssh fact@161.72.93.131 "ls -lH ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizedaq=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         if ! [ ${dl00[1]} -eq -1 ]
         then
            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${archive[1]} -eq -1 ]
         then
            sizearchive=( `ls -lH $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${fails[1]} -eq -1 ]
         then
            sizefails=( `ls -lH $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${wue[1]} -eq -1 ]
         then
            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         fi
         #if ! [ ${phido[1]} -eq -1 ]
         #then
         #   sizephido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "ls -l $phidorawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         #fi
         
         # check file sizes for run
         # lp
         if ! [ "$sizenewdaq" = "$sizedaq" ]
         then
            printprocesslog "WARN "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")" >> $logfile 2>&1
            daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
         else
            daqokcounter=`echo " $daqokcounter + 1 " | bc -l `
         fi
         # dl00
         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
         then
            printprocesslog "WARN "$rawfile2" data("$sizezip") dl00("$sizedl00")"
            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")" >> $logfile 2>&1
            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
         else
            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
         fi
         #if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1  ]
         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 -a "$sizearchive" != "" ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1 -a "$sizefails" != "" ]
         then
            printprocesslog "WARN "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")" >> $logfile 2>&1
            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
         else
            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
         fi
         # wue
         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
         then
            printprocesslog "WARN "$rawfile2" data("$sizezip") wue("$sizewue")"
            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")" >> $logfile 2>&1
            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
         else
            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
         fi
         ## phido
         #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
         #then
         #   printprocesslog "WARN "$rawfile2" data("$sizezip") phido("$sizephido")"
         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")" >> $logfile 2>&1
         #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
         #else
         #   phidookcounter=`echo " $phidookcounter + 1 " | bc -l `
         #fi
      done
      
      #result=$result"-"
      # raw files
      if [ $daqokcounter -eq ${daq[0]} ]
      then
         numok=`echo " $numok + 1 " | bc -l `
         result4="0"
      else
         result4="1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # zipped files
      # dl00
      if [ $dl00okcounter -eq ${dl00[0]} ]
      then
         result4=$result4"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result4=$result4"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # archive
      # daq had been used, because archive[0] doesn't include fails[0]
      #if [ $archiveokcounter -eq ${newdaq[0]} ]
      # archive[0] had been used, because newdaq[0] might be empty 
      #   in case the data was taken on data
      #if [ $archiveokcounter -eq ${archive[0]} ]
      # compare with daq[0] as there should be always data on data
      if [ $archiveokcounter -eq ${daq[0]} ]
      then
         result4=$result4"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result4=$result4"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # wue
      if [ $wueokcounter -eq ${wue[0]} ]
      then
         result4=$result4"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result4=$result4"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      ## phido
      #if [ $phidookcounter -eq ${phido[0]} ]
      #then
      #   result4=$result4"0"
      #   numok=`echo " $numok + 1 " | bc -l `
      #else
      #   result4=$result4"1"
      #   numpb=`echo " $numpb + 1 " | bc -l `
      #fi
      printprocesslog "INFO "$daqokcounter" files are ok on daq (raw)."
      printprocesslog "INFO "$dl00okcounter" files are ok on dl00."
      printprocesslog "INFO "$wueokcounter" files are ok in Wue."
      printprocesslog "INFO "$archiveokcounter" files are ok in the archive."
      #printprocesslog "INFO "$phidookcounter" files are ok on Phido."
      echo "INFO "$daqokcounter" files are ok on daq (raw)." >> $logfile 2>&1
      echo "INFO "$dl00okcounter" files are ok on dl00." >> $logfile 2>&1
      echo "INFO "$wueokcounter" files are ok in Wue." >> $logfile 2>&1
      echo "INFO "$archiveokcounter" files are ok in the archive." >> $logfile 2>&1
      #echo "INFO "$phidookcounter" files are ok on Phido." >> $logfile 2>&1
      if [ $daqdiffcounter -gt 0 ]
      then 
         printprocesslog "WARN "$daqdiffcounter" files have a different size on daq (raw)."
         echo "WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
      else
         printprocesslog "INFO "$daqdiffcounter" files have a different size on daq (raw)."
         echo "INFO "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
      fi
      if [ $dl00diffcounter -gt 0 ]
      then 
         printprocesslog "WARN "$dl00diffcounter" files have a different size on dl00."
         echo "WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
      else
         printprocesslog "INFO "$dl00diffcounter" files have a different size on dl00."
         echo "INFO "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
      fi
      if [ $wuediffcounter -gt 0 ]
      then 
         printprocesslog "WARN "$wuediffcounter" files have a different size in Wue."
         echo "WARN "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
      else
         printprocesslog "INFO "$wuediffcounter" files have a different size in Wue."
         echo "INFO "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
      fi
      if [ $archivediffcounter -gt 0 ]
      then 
         printprocesslog "WARN "$archivediffcounter" files have a different size in the archive."
         echo "WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
      else
         printprocesslog "INFO "$archivediffcounter" files have a different size in the archive."
         echo "INFO "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
      fi
      #printprocesslog "WARN "$phidodiffcounter" files have a different size on Phido."
      #echo "WARN "$phidodiffcounter" files have a different size on Phido." >> $logfile 2>&1
   fi
   
   # print summary: 
   printprocesslog "INFO day ok: "$numdaysok
   printprocesslog "INFO numok: "$numok
   printprocesslog "INFO numpb: "$numpb
   printprocesslog "result:"
   printprocesslog "(qla-#files-dudir-db-filesize)"
   #printprocesslog " ldawp-ldawp-ldawp-ldawp"
   printprocesslog " q-ldaw-ldaw-ldaw-ldaw"
   printprocesslog " "$result1"-"$result2"-"$result3"-"$result4
   echo "INFO day ok: "$numdaysok >> $logfile 2>&1
   echo "INFO numok: "$numok >> $logfile 2>&1
   echo "INFO numpb: "$numpb >> $logfile 2>&1
   echo "result:" >> $logfile 2>&1
   echo "(qla-#files-dudir-db-filesize)" >> $logfile 2>&1
   #echo " ldawp-ldawp-ldawp-ldawp" >> $logfile 2>&1
   echo " q-ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1
   echo " "$result1"-"$result2"-"$result3"-"$result4 >> $logfile 2>&1
   sumdatanew=`echo " ( ${daq[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l | cut -d. -f1`
   sumdata=`echo " $sumdata + $sumdatanew " | bc -l | cut -d. -f1`
   printprocesslog "checked alread "$sumdata" GB. "$sumdatanew" "${daq[1]}" "${zip[1]}
   if [ $numpb -lt 4 ]
   then 
      numdaysok=`echo " $numdaysok + 1 " | bc -l `
   fi

   echo "SUMMARY for "$date
   echo "-----------------------"
   echo "SUMMARY for "$date >> $logfile 2>&1
   echo "-----------------------" >> $logfile 2>&1
   echo "SUMMARY for "$date >> $logfile2 2>&1
   echo "-----------------------" >> $logfile2 2>&1
   #echo "res1:"$result1
   #echo "res3:"$result3
   #echo "res4:"$result4
   #echo "arch:"${archive[0]}
   #echo "isdc:"$numisdc
   if [ "$result1" = "0-0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ]
   then 
      echo "  EVERYTHING is ok. "$date" can be deleted. "
      echo "" >> $logfile 2>&1
      echo "  EVERYTHING is ok. "$date" can be deleted. " >> $logfile 2>&1
      echo "  EVERYTHING is ok. "$date" can be deleted. " >> $logfile2 2>&1
      echo "    Details in the logfile "$logfile
      echo "" >> $logfile 2>&1
      echo "    Details in the logfile "$logfile >> $logfile2 2>&1
      sendemail="yes"
   else
      if [ "$result1" = "0-0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ]
      then 
         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. "
         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " >> $logfile 2>&1
         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " >> $logfile2 2>&1
         echo "  TRANSFER is ok. "$date" can be deleted. "
         echo "" >> $logfile 2>&1
         echo "  TRANSFER is ok. "$date" can be deleted. " >> $logfile 2>&1
         echo "  TRANSFER is ok. "$date" can be deleted. " >> $logfile2 2>&1
         echo "    Details in the logfile "$logfile
         echo "" >> $logfile 2>&1
         echo "    Details in the logfile "$logfile >> $logfile2 2>&1
         sendemail="yes"
      else
         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile
         echo "" >> $logfile 2>&1
         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile 2>&1
         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile2 2>&1
         echo "    resetting jobs in the DB might be needed."
         echo "    resetting jobs in the DB might be needed." >> $logfile 2>&1
         echo "    resetting jobs in the DB might be needed." >> $logfile2 2>&1
      fi
   fi
   if [ $daqdiffcounter -gt 0 ]
   then 
      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)."
      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile2 2>&1
   fi
   if [ $dl00diffcounter -gt 0 ]
   then 
      echo "  WARN "$dl00diffcounter" files have a different size on dl00."
      echo "  WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
      echo "  WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile2 2>&1
   fi
   if [ $wuediffcounter -gt 0 ]
   then 
      echo "  WARN "$wuediffcounter" files have a different size in Wue."
      echo "  WARN "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
      echo "  WARN "$wuediffcounter" files have a different size in Wue." >> $logfile2 2>&1
   fi
   if [ $archivediffcounter -gt 0 ]
   then 
      echo "  WARN "$archivediffcounter" files have a different size in the archive."
      echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
      echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile2 2>&1
   fi
   if ! [ $numdatruns -eq $qla ]
   then
      echo "WARN not all data runs are processed yet by the QLA. " 
      echo "WARN not all data runs are processed yet by the QLA. " >> $logfile 2>&1
      echo "WARN not all data runs are processed yet by the QLA. " >> $logfile2 2>&1
   fi
   echo ""
   echo "" >> $logfile 2>&1
   echo "" >> $logfile2 2>&1
done

#sendemail="yes"
if [ "$sendemail" = "yes" ]
then
   echo ""
   echo "INFO send email with "$logfile2"to shift@fact-project.org "
   printprocesslog "INFO send email with "$logfile2"to shift@fact-project.org "
   cat $logfile2 | mail -s "testmail for info on deleting data" shift@fact-project.org
fi 

printprocesslog "INFO finished $0"