#!/bin/bash
#
# This script checks whether data can be deleted
#

source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"

ssh fact@161.72.93.131 "df -h /*da*"

df -h /scratch

# get last 10 nights (skip current night)
dates=( `date +%Y/%m/%d --date="-360hour"` `date +%Y/%m/%d --date="-336hour"` `date +%Y/%m/%d --date="-312hour"` \ 
        `date +%Y/%m/%d --date="-288hour"` `date +%Y/%m/%d --date="-264hour"` `date +%Y/%m/%d --date="-240hour"` \ 
        `date +%Y/%m/%d --date="-216hour"` `date +%Y/%m/%d --date="-192hour"` `date +%Y/%m/%d --date="-168hour"` \ 
        `date +%Y/%m/%d --date="-144hour"` `date +%Y/%m/%d --date="-120hour"` `date +%Y/%m/%d --date="-96hour"` \
        `date +%Y/%m/%d --date="-72hour"` `date +%Y/%m/%d --date="-48hour"` `date +%Y/%m/%d --date="-24hour"` \
        )

dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` )

numdaysok=0
numdaysoklimit=3
sumdata=0
checklimit=2500
for date in ${dates[@]}
do 
   echo ""
   echo ""
   echo ""
   echo $date

   # some counters
   numdiff=0
   numok=0
   numpb=0
   
   # check always only $numdaysoklimit days
   # and require at least $checklimit GB that have been checked 
   #  remark: bc: expr1 < expr2: the result is 1 if expr1 is strictly less than expr2
   if [ $numdaysok -ge $numdaysoklimit ] && [ $(echo " $sumdata > $checklimit " | bc -l) -eq 1 ]
   then
      echo "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
      continue
   fi
   
   # get paths
   date2=`echo $date | sed -e 's/\///g'`
   lprawpath="/daq/raw/"$date
   lprawpath2="/loc_data/raw/"$date
   lpziprawpath="/loc_data/zipraw/"$date
   localrawpath="/scratch/from_lapalma/raw/"$date
   localrawpath3="/fact/raw/"$date
   localfailpath="/archive/fact/fails/raw/"$date
   wuerawpath="/fact/raw/"$date
   
   # get disk usage and number of files for directory
   daq=( `ssh fact@161.72.93.131 "if [ -d $lprawpath ]; then ls $lprawpath/* | wc -l; du -s -b --apparent-size $lprawpath; else echo '-1 -1 -1'; fi"` )
   data=( `ssh fact@161.72.93.131 "if [ -d $lprawpath2 ]; then ls $lprawpath2/* | wc -l; du -s -b --apparent-size $lprawpath2; else echo '-1 -1 -1'; fi"` )
   if [ ${daq[0]} -eq -1 ] && [ ${data[0]} -eq -1 ] 
   then
      echo "INFO no data available on daq for "$date
      continue
   fi
   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
   dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* | wc -l; du -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
   fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
   wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
   query="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM RunInfo WHERE fNight="$date2
   querystart="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM "
   queryjoin="LEFT JOIN RunInfo USING(fNight,fRunID) "
   querywhere="WHERE fNight="$date2" AND NOT ISNULL(fStartTime) AND NOT ISNULL(fStopTime) AND ISNULL(fReturnCode)"
   numruns=`sendquery`
   if [ "$numruns" == "" ]
   then
      numruns=0
   fi
   query=$querystart"RawFileRsyncedISDCStatus "$queryjoin" "$querywhere
   numrsynced=`sendquery`
   if [ "$numrsynced" == "" ]
   then
      numrsynced=0
   fi
   query=$querystart"RawFileAvailISDCStatus "$queryjoin" "$querywhere
   numisdc=`sendquery`
   if [ "$numisdc" == "" ]
   then
      numisdc=0
   fi
   query=$querystart"RawFileAvailWueStatus "$queryjoin" "$querywhere
   numwue=`sendquery`
   if [ "$numwue" == "" ]
   then
      numwue=0
   fi
   echo "disk: "
   echo "  daq     "${daq[@]}
   echo "  data    "${data[@]}
   echo "  zip     "${zip[@]}
   echo "  dl00    "${dl00[@]}
   echo "  wue     "${wue[@]}
   echo "  arch    "${archive[@]}
   echo "  fail    "${fails[@]}
   echo "db: "
   echo "  runinfo "$numruns
   echo "  rsynced "$numrsynced
   echo "  isdc    "$numisdc
   echo "  wue     "$numwue
   
   # check if file are available in the different places
   if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] 
   then
      echo "INFO data not available on /scratch on dl00 for "$date
   fi
   if [ ${archive[0]} -eq -1 ] 
   then
      echo "INFO data not in archive for "$date
   fi
   
   # check if number of files agree in the different places
   # archive
   if ! [ ${archive[0]} -eq -1 ]  && ! [ ${archive[0]} -eq ${daq[0]} ]
   then
      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
      #check /archive/rev_1/failed
      if ! [ ${fails[0]} -eq -1 ]
      then
         sum=`echo " ${fails[0]} + ${archive[0]} " | bc -l `
         if ! [ $sum -eq ${daq[0]} ]
         then
            echo "ERROR number of files in archive ("$sum") is different from number of files in La Palma ("${daq[0]}")."
            numpb=`echo " $numpb + 1 " | bc -l `
            result="1"
         else
            numok=`echo " $numok + 1 " | bc -l `
            result="0"
         fi
      else
         result="1"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result="0"
   fi
   # lp
   if ! [ ${data[0]} -eq -1 ]  && ! [ ${data[0]} -eq ${daq[0]} ]
   then
      echo "WARN number of files on data (" ${data[0]}") does not agree with number of files on daq (" ${daq[0]}") for "$date
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # dl00
   if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${daq[0]} ]
   then
      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # wue
   if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${daq[0]} ]
   then
      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   
   result=$result"-"
   # check du for raw files 
   # la palma
   if ! [ ${daq[1]} -eq ${data[1]} ]
   then
      echo "WARN size of data doesn't agree on daq ("${daq[1]}") and data ("${data[1]}") for "$date
      numdiff=`echo " $numdiff + 1 " | bc -l `
      numpb=`echo " $numpb + 1 " | bc -l `
      result=$result"1"
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # check du for zipped raw files 
   # dl00
   if ! [ ${zip[1]} -eq ${dl00[1]} ] && [ $date2 -gt 20120307 ]
   then
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
      if ! [ ${dl00[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # archive
   if ! [ ${zip[1]} -eq ${archive[1]} ]
   then
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
      if ! [ ${archive[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   # wue
   if ! [ ${zip[1]} -eq ${wue[1]} ]
   then
      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
      if ! [ ${wue[1]} -eq -1 ]
      then 
         numdiff=`echo " $numdiff + 1 " | bc -l `
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   else
      numok=`echo " $numok + 1 " | bc -l `
      result=$result"0"
   fi
   result=$result"-"

   # check DB (only starting from 8.3.2012) (if-clause to be removed later)
   if [ $date2 -gt 20120307 ]
   then
      # lp
      if ! [ $numruns -eq ${daq[0]} ]
      then
         echo "WARN number of runs on daq ("${daq[0]}") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      # dl00
      if ! [ $numruns -eq $numrsynced ]
      then
         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      # archive
      if ! [ $numruns -eq $numisdc ]
      then
         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
      # wue
      if ! [ $numruns -eq $numwue ]
      then
         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
         numpb=`echo " $numpb + 1 " | bc -l `
         result=$result"1"
      else
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      fi
   fi
   
   #numdiff=0 # add for debugging so that single file sizes are not checked
   echo "numdiff: "$numdiff
   echo "INFO numok: "$numok
   echo "INFO numpb: "$numpb
   if [ $numdiff -gt 0 ]
   then
      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2
      runs=( `sendquery` )
      archivediffcounter=0
      archiveokcounter=0
      wuediffcounter=0
      wueokcounter=0
      dl00diffcounter=0
      dl00okcounter=0
      datadiffcounter=0
      dataokcounter=0
      echo "INFO found "${#runs[@]}" rawfiles in DB."
      for run in ${runs[@]}
      do 
         rawfile=$date2"_"`printf %03d $run`".fits"
         rawfile2=$rawfile".gz"
         
         # get file sizes for run
         sizedaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizedata=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         if ! [ ${dl00[1]} -eq -1 ]
         then
            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${archive[1]} -eq -1 ]
         then
            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${fails[1]} -eq -1 ]
         then
            sizefails=( `ls -l $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${wue[1]} -eq -1 ]
         then
            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         fi
         
         # check file sizes for run
         # lp
         if ! [ "$sizedaq" = "$sizedata" ]
         then
            echo "  "$rawfile" daq("$sizedaq") data("$sizedata")"
            datadiffcounter=`echo " $datadiffcounter + 1 " | bc -l `
         else
            dataokcounter=`echo " $dataokcounter + 1 " | bc -l `
         fi
         # dl00
         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
         else
            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
         fi
         # archive
         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1  ]
         then
            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
         else
            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
         fi
         # wue
         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
         else
            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
         fi
      done
      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
      drsruns=( `sendquery` )
      echo "INFO found "${#drsruns[@]}" drsfiles in DB."
      for drsrun in ${drsruns[@]}
      do 
         rawfile=$date2"_"`printf %03d $run`"drs.fits"
         rawfile2=$rawfile".gz"
         # get file sizes for run
         sizedaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizedata=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         if ! [ ${dl00[1]} -eq -1 ]
         then
            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${archive[1]} -eq -1 ]
         then
            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${fails[1]} -eq -1 ]
         then
            sizefails=( `ls -l $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
         fi
         if ! [ ${wue[1]} -eq -1 ]
         then
            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
         fi
         
         # check file sizes for run
         # lp
         if ! [ "$sizedaq" = "$sizedata" ]
         then
            echo "  "$rawfile" daq("$sizedaq") data("$sizedata")"
            datadiffcounter=`echo " $datadiffcounter + 1 " | bc -l `
         else
            dataokcounter=`echo " $dataokcounter + 1 " | bc -l `
         fi
         # dl00
         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
         else
            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
         fi
         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1  ]
         then
            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
         else
            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
         fi
         # wue
         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
         then
            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
         else
            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
         fi
      done
      
      result=$result"-"
      # raw files
      if [ $dataokcounter -eq ${data[0]} ]
      then
         numok=`echo " $numok + 1 " | bc -l `
         result=$result"0"
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # zipped files
      # dl00
      if [ $dl00okcounter -eq ${dl00[0]} ]
      then
         result=$result"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # archive
      if [ $archiveokcounter -eq ${daq[0]} ]
      then
         result=$result"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      # wue
      if [ $wueokcounter -eq ${wue[0]} ]
      then
         result=$result"0"
         numok=`echo " $numok + 1 " | bc -l `
      else
         result=$result"1"
         numpb=`echo " $numpb + 1 " | bc -l `
      fi
      echo "INFO "$dataokcounter" files are ok on data (raw)."
      echo "INFO "$dl00okcounter" files are ok on dl00."
      echo "INFO "$wueokcounter" files are ok in Wue."
      echo "INFO "$archiveokcounter" files are ok in the archive."
      echo "WARN "$datadiffcounter" files have a different size on data (raw)."
      echo "WARN "$dl00diffcounter" files have a different size on dl00."
      echo "WARN "$wuediffcounter" files have a different size in Wue."
      echo "WARN "$archivediffcounter" files have a different size in the archive."
   fi
   
   # print summary: 
   echo "INFO day ok: "$numdaysok
   echo "INFO numok: "$numok
   echo "INFO numpb: "$numpb
   echo "result:"
   echo "(#files-dudir-db-filesize)"
   echo " ldaw-ldaw-ldaw-ldaw"
   echo " "$result
   sumdatanew=`echo " ( ${data[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l `
   sumdata=`echo " $sumdata + $sumdatanew " | bc -l `
   echo "checked alread "$sumdata" GB."$sumdatanew" "${data[1]}" "${zip[1]}
   if [ $numpb -lt 4 ]
   then 
      numdaysok=`echo " $numdaysok + 1 " | bc -l `
   fi
done

printprocesslog "INFO finished $0"