#!/bin/bash
#
# This script checks whether data can be deleted
#

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"

source /home_nfs/isdc/fact_opr/myagent.sh

# check first the disk in LP and on dl00
ssh fact@161.72.93.131 "df -h /*da*"
df -h /scratch
echo ""

# check next the DB to know if some transfer processes failed or crashed
function check_runs_in_db()
{
   query="SELECT "$toquery" FROM "$1" "$where
   #echo $query
   runs=( `sendquery $query` )
   if [ ${#runs[@]} -gt 0 ]
   then
      #for run in ${runs[@]}
      #do
      #   echo $run
      #done
      echo -e "\e[1;31m\x1b[5m ==>\e[00m "$1": "${runs[@]}"\e[1;31m\x1b[5m <==\e[00m "
      echo "SELECT fNight, fRunId FROM "$1" "$where";"
      echo "UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";"
   fi
}
# get information of runs where transfer had a problem
toquery="fNight, fRunID, fStartTime, fStopTime, fAvailable, fProcessingSiteKey, fReturnCode "
toquery="CONCAT(fNight, '_', fRunID, '(', fStartTime, '-', fStopTime, ':', fReturnCode, ')') "
toquery="CONCAT(fNight, '_', fRunID, ':', fReturnCode) "
toquery="IF (ISNULL(fReturnCode), CONCAT(fNight, '_', fRunID, 'crashed'), CONCAT(fNight, '_', fRunID, 'failed', fReturnCode)) "
where="WHERE NOT ISNULL(fReturnCode) OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) AND fStartTime < DATE_ADD(Now(), INTERVAL -5 HOUR) "
check_runs_in_db "RawFileRsyncedISDCStatus"
check_runs_in_db "RawFileAvailWueStatus"
# RawFileAvailISDC needs a different treatment 
#  as return code 0 means that file is in fails folder in archive
where="WHERE fReturnCode>0 OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) "
check_runs_in_db "RawFileAvailISDCStatus"


# get last 10 nights (skip current night)
dates=( `date +%Y/%m/%d --date="-360hour"` `date +%Y/%m/%d --date="-336hour"` `date +%Y/%m/%d --date="-312hour"` \ 
        `date +%Y/%m/%d --date="-288hour"` `date +%Y/%m/%d --date="-264hour"` `date +%Y/%m/%d --date="-240hour"` \ 
        `date +%Y/%m/%d --date="-216hour"` `date +%Y/%m/%d --date="-192hour"` `date +%Y/%m/%d --date="-168hour"` \ 
        `date +%Y/%m/%d --date="-144hour"` `date +%Y/%m/%d --date="-120hour"` `date +%Y/%m/%d --date="-96hour"` \
        `date +%Y/%m/%d --date="-72hour"` `date +%Y/%m/%d --date="-48hour"` `date +%Y/%m/%d --date="-24hour"` \
        )
# get nights from directory in LP
dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` )

#dates=( "2013/04/18" )
#short="yes"

numdaysok=0
numdaysoklimit=5
sumdata=0
checklimit=5000
for date in ${dates[@]}
do 
   echo ""
   echo ""
   echo ""
   echo $date

   # some counters
   numdiff=0
   numok=0
   numpb=0
   
   # check always only $numdaysoklimit days
   # and require at least $checklimit GB that have been checked 
   #  remark: bc: expr1 < expr2: the result is 1 if expr1 is strictly less than expr2
   if [ $numdaysok -ge $numdaysoklimit ] && [ $(echo " $sumdata > $checklimit " | bc -l) -eq 1 ]
   then
      echo "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
      continue
   fi
   
   # get paths
   date2=`echo $date | sed -e 's/\///g'`
#   lprawpath="/daq/raw/"$date
   lprawpath="/newdaq/raw/"$date
#   lprawpath2="/loc_data/raw/"$date
   lprawpath2="/daq/raw/"$date
   lpziprawpath="/loc_data/zipraw/"$date
   localrawpath="/scratch/from_lapalma/raw/"$date
   localrawpath3="/fact/raw/"$date
#   localfailpath="/archive/fact/fails/raw/"$date
   localfailpath="/gpfs/fact/fact-archive/fails/raw/"$date
   wuerawpath="/fact/raw/"$date
   #phidorawpath="/fhgfs/groups/app/fact-construction/raw/"$date
   
   # get disk usage and number of files for directory
   newdaq=( `ssh fact@161.72.93.131 "if [ -d $lprawpath ]; then ls $lprawpath/* | wc -l; du -s -b --apparent-size $lprawpath; else echo '-1 -1 -1'; fi"` )
   daq=( `ssh fact@161.72.93.131 "if [ -d $lprawpath2 ]; then ls $lprawpath2/* | wc -l; du -s -b --apparent-size $lprawpath2; else echo '-1 -1 -1'; fi"` )
   if [ ${newdaq[0]} -eq -1 ] && [ ${daq[0]} -eq -1 ] 
   then
      echo "INFO no data available on newdaq for "$date
      continue
   fi
   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
   dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* | wc -l; du -L -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
   fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -L -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
   wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
   #phido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "if [ -d $phidorawpath ]; then ls $phidorawpath/* | wc -l; du -s -b --apparent-size $phidorawpath; else echo '-1 -1 -1'; fi"` )
   query="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM RunInfo WHERE fNight="$date2
   querystart="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM "
   queryjoin="LEFT JOIN RunInfo USING(fNight,fRunID) "
   querywhere="WHERE fNight="$date2" AND NOT ISNULL(fStartTime) AND NOT ISNULL(fStopTime) AND ISNULL(fReturnCode)"
   numruns=`sendquery`
   if [ "$numruns" == "" ]
   then
      numruns=0
   fi
   query=$querystart"RawFileRsyncedISDCStatus "$queryjoin" "$querywhere
   numrsynced=`sendquery`
   if [ "$numrsynced" == "" ]
   then
      numrsynced=0
   fi
   query=$querystart"RawFileAvailISDCStatus "$queryjoin" "$querywhere
   numisdc=`sendquery`
   if [ "$numisdc" == "" ]
   then
      numisdc=0
   fi
   query=$querystart"RawFileAvailWueStatus "$queryjoin" "$querywhere
   numwue=`sendquery`
   if [ "$numwue" == "" ]
   then
      numwue=0
   fi
   #query=$querystart"RawFileAvailPhidoStatus "$queryjoin" "$querywhere
   #numphido=`sendquery`
   #if [ "$numphido" == "" ]
   #then
   #   numphido=0
   #fi
   echo "disk: "
   echo "  newdaq  "${newdaq[@]}
   echo "  daq     "${daq[@]}
   echo "  data    "${zip[@]}
   echo "  dl00    "${dl00[@]}
   echo "  wue     "${wue[@]}
   echo "  arch    "${archive[@]}
   echo "  fail    "${fails[@]}
   #echo "  phido   "${phido[@]}
   echo "db: "
   echo "  runinfo "$numruns
   echo "  rsynced "$numrsynced
   echo "  isdc    "$numisdc
   echo "  wue     "$numwue
   #echo "  phido   "$numphido
   
   # check if file are available in the different places
   if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] 
   then
      echo "INFO data not available on /scratch on dl00 for "$date
   fi
   if [ ${archive[0]} -eq -1 ] 
   then
      echo "INFO data not in archive for "$date
   fi
   
   # check if number of files agree in the different places
   # lp
   if ! [ ${daq[0]} -eq -1 ]  && ! [ ${daq[0]} -eq ${newdaq[0]} ]
   then
      echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date
      numpb=`echo " $numpb + 1 " | bc -l `
      result="1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result="0"
   fi
   # dl00
   if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${newdaq[0]} ]
   then
      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # archive
   if ! [ ${archive[0]} -eq -1 ]  && ! [ ${archive[0]} -eq ${newdaq[0]} ]
   then
      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
      #check /archive/rev_1/failed
      if ! [ ${fails[0]} -eq -1 ]
      then
         sum=`echo " ${fails[0]} + ${archive[0]} " | bc -l `
         if ! [ $sum -eq ${newdaq[0]} ]
         then
            echo "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")."
            numpb=`echo " $numpb + 1 " | bc -l `
            result=$result"1"
         else
            numok=`echo " $numok + 1 " | bc -l `
            result=$result"0"
         fi
      else
         result=$result"1"
      fi
   else
      if [ ${archive[0]} -eq -1 ]
      then 
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   fi
   # wue
   if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${newdaq[0]} ]
   then
      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   ## phido
   #if ! [ ${phido[0]} -eq -1 ]  && ! [ ${phido[0]} -eq ${newdaq[0]} ]
   #then
   #   echo "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
   #   numpb=`echo " $numpb + 1 " | bc -l `
   #   result=$result"1"
   #else
   #   numok=`echo " $numok + 1 " | bc -l `
   #   result=$result"0"
   #fi
   
   if [ "$short" = "yes" ] || [ "$result" != "0000" ]
   then
      echo $result
      continue
   fi
   
   result=$result"-"
   # check du for raw files 
   # la palma
   if ! [ ${newdaq[1]} -eq ${daq[1]} ]
   then
      echo "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date
      numdiff=`echo " $numdiff + 1 " | bc -l `
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # check du for zipped raw files 
   # dl00
   if ! [ ${zip[1]} -eq ${dl00[1]} ] && [ $date2 -gt 20120307 ]
   then
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
      if ! [ ${dl00[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # archive
   if ! [ ${zip[1]} -eq ${archive[1]} ]
   then
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
      if ! [ ${archive[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # wue
   if ! [ ${zip[1]} -eq ${wue[1]} ]
   then
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
      if ! [ ${wue[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   ## phido
   #if ! [ ${zip[1]} -eq ${phido[1]} ]
   #then
   #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date
   #   if ! [ ${phido[1]} -eq -1 ]
   #   then 
   #      numdiff=`echo " $numdiff + 1 " | bc -l `
   #      numpb=`echo " $numpb + 1 " | bc -l `
   #      result=$result"1"
   #   else
   #      numok=`echo " $numok + 1 " | bc -l `
   #      result=$result"0"
   #   fi
   #else
   #   numok=`echo " $numok + 1 " | bc -l `
   #   result=$result"0"
   #fi
   result=$result"-"

   # check DB (only starting from 8.3.2012) (if-clause to be removed later)
   if [ $date2 -gt 20120307 ]
   then
      # lp
      if ! [ $numruns -eq ${newdaq[0]} ]
      then
         echo "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      # dl00
      if ! [ $numruns -eq $numrsynced ]
      then
         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      # archive
      if ! [ $numruns -eq $numisdc ]
      then
         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      # wue
      if ! [ $numruns -eq $numwue ]
      then
         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      ## phido
      #if ! [ $numruns -eq $numphido ]
      #then
      #   echo "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")"
      #   numpb=`echo " $numpb + 1 " | bc -l `
      #   result=$result"1"
      #else
      #   numok=`echo " $numok + 1 " | bc -l `
      #   result=$result"0"
      #fi
   fi
   
   #numdiff=0 # add for debugging so that single file sizes are not checked
   echo "numdiff: "$numdiff
   echo "INFO numok: "$numok
   echo "INFO numpb: "$numpb
   #if [ $numdiff -gt 0 ]
   if [ $numdiff -ge 0 ]
   then
      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2
      runs=( `sendquery` )
      archivediffcounter=0
      archiveokcounter=0
      wuediffcounter=0
      wueokcounter=0
      #phidodiffcounter=0
      #phidookcounter=0
      dl00diffcounter=0
      dl00okcounter=0
      daqdiffcounter=0
      daqokcounter=0
      echo "INFO found "${#runs[@]}" rawfiles in DB."
      for run in ${runs[@]}
      do 
         rawfile=$date2"_"`printf %03d $run`".fits"
         rawfile2=$rawfile".gz"
         
         # get file sizes for run
         sizenewdaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizedaq=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         if ! [ ${dl00[1]} -eq -1 ]
         then
            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${archive[1]} -eq -1 ]
         then
            sizearchive=( `ls -lH $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${fails[1]} -eq -1 ]
         then
            sizefails=( `ls -lH $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${wue[1]} -eq -1 ]
         then
            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         fi
#         echo "vgl "$run": "$sizezip" - "$sizearchive
         #if ! [ ${phido[1]} -eq -1 ]
         #then
         #   sizephido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "ls -l $phidorawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         #fi
         
         # check file sizes for run
         # lp
         if ! [ "$sizenewdaq" = "$sizedaq" ]
         then
            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
            daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
         else
            daqokcounter=`echo " $daqokcounter + 1 " | bc -l `
         fi
         # dl00
         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
         else
            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
         fi
         # archive
         #if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 -a "$sizearchive" != "" ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1 -a "$sizefails" != "" ] #not yet ingested files are treated wrongly
         #if [ ${archive[1]} -ne -1 ] && [ "$sizezip" != "$sizearchive" -o "$sizezip" != "$sizefails" ] 
         if [ ${archive[1]} -ne -1 -a "$sizezip" != "$sizearchive" -a "$sizezip" != "$sizefails" ] 
         then
            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
            echo " "$sizezip"-"$sizearchive"-"${archive[1]}"-"$sizezip"-"$sizefails"-"${fails[1]}
            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
         else
            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
         fi
         # wue
         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
         else
            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
         fi
         ## phido
         #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
         #then
         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")"
         #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
         #else
         #   phidookcounter=`echo " $phidookcounter + 1 " | bc -l `
         #fi
      done
      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
      drsruns=( `sendquery` )
      echo "INFO found "${#drsruns[@]}" drsfiles in DB."
      for drsrun in ${drsruns[@]}
      do 
         rawfile=$date2"_"`printf %03d $drsrun`".drs.fits"
         rawfile2=$rawfile".gz"
         # get file sizes for run
         sizenewdaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizedaq=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         if ! [ ${dl00[1]} -eq -1 ]
         then
            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${archive[1]} -eq -1 ]
         then
            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${fails[1]} -eq -1 ]
         then
            sizefails=( `ls -l $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${wue[1]} -eq -1 ]
         then
            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         fi
         #if ! [ ${phido[1]} -eq -1 ]
         #then
         #   sizephido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "ls -l $phidorawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         #fi
         
         # check file sizes for run
         # lp
         if ! [ "$sizenewdaq" = "$sizedaq" ]
         then
            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
            daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
         else
            daqokcounter=`echo " $daqokcounter + 1 " | bc -l `
         fi
         # dl00
         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
         else
            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
         fi
         #if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1  ]
         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 -a "$sizearchive" != "" ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1 -a "$sizefails" != "" ]
         then
            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
         else
            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
         fi
         # wue
         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
         else
            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
         fi
         ## phido
         #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
         #then
         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")"
         #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
         #else
         #   phidookcounter=`echo " $phidookcounter + 1 " | bc -l `
         #fi
      done
      
      result=$result"-"
      # raw files
      if [ $daqokcounter -eq ${daq[0]} ]
      then
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # zipped files
      # dl00
      if [ $dl00okcounter -eq ${dl00[0]} ]
      then
         result=$result"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # archive
      # daq had been used, because archive[0] doesn't include fails[0]
      #if [ $archiveokcounter -eq ${newdaq[0]} ]
      # archive[0] had been used, because newdaq[0] might be empty 
      #   in case the data was taken on data
      #if [ $archiveokcounter -eq ${archive[0]} ]
      # compare with daq[0] as there should be always data on data
      if [ $archiveokcounter -eq ${daq[0]} ]
      then
         result=$result"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # wue
      if [ $wueokcounter -eq ${wue[0]} ]
      then
         result=$result"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      ## phido
      #if [ $phidookcounter -eq ${phido[0]} ]
      #then
      #   result=$result"0"
      #   numok=`echo " $numok + 1 " | bc -l `
      #else
      #   result=$result"1"
      #   numpb=`echo " $numpb + 1 " | bc -l `
      #fi
      echo "INFO "$daqokcounter" files are ok on daq (raw)."
      echo "INFO "$dl00okcounter" files are ok on dl00."
      echo "INFO "$wueokcounter" files are ok in Wue."
      echo "INFO "$archiveokcounter" files are ok in the archive."
      #echo "INFO "$phidookcounter" files are ok on Phido."
      echo "WARN "$daqdiffcounter" files have a different size on daq (raw)."
      echo "WARN "$dl00diffcounter" files have a different size on dl00."
      echo "WARN "$wuediffcounter" files have a different size in Wue."
      echo "WARN "$archivediffcounter" files have a different size in the archive."
      #echo "WARN "$phidodiffcounter" files have a different size on Phido."
   fi
   
   # print summary: 
   echo "INFO day ok: "$numdaysok
   echo "INFO numok: "$numok
   echo "INFO numpb: "$numpb
   echo "result:"
   echo "(#files-dudir-db-filesize)"
   #echo " ldawp-ldawp-ldawp-ldawp"
   echo " ldaw-ldaw-ldaw-ldaw"
   echo " "$result
   sumdatanew=`echo " ( ${daq[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l | cut -d. -f1`
   sumdata=`echo " $sumdata + $sumdatanew " | bc -l | cut -d. -f1`
   echo "checked alread "$sumdata" GB. "$sumdatanew" "${daq[1]}" "${zip[1]}
   if [ $numpb -lt 4 ]
   then 
      numdaysok=`echo " $numdaysok + 1 " | bc -l `
   fi
done

printprocesslog "INFO finished $0"