Index: /trunk/DataCheck/Transfer/CheckTransfer.sh
===================================================================
--- /trunk/DataCheck/Transfer/CheckTransfer.sh	(revision 13154)
+++ /trunk/DataCheck/Transfer/CheckTransfer.sh	(revision 13154)
@@ -0,0 +1,513 @@
+#!/bin/bash
+#
+# This script checks whether data can be transferred
+#
+
+source `dirname $0`/../Sourcefile.sh
+printprocesslog "INFO starting $0"
+
+# get last 10 nights (skip current night)
+dates=( `date +%Y/%m/%d --date="-360hour"` `date +%Y/%m/%d --date="-336hour"` `date +%Y/%m/%d --date="-312hour"` \ 
+        `date +%Y/%m/%d --date="-288hour"` `date +%Y/%m/%d --date="-264hour"` `date +%Y/%m/%d --date="-240hour"` \ 
+        `date +%Y/%m/%d --date="-216hour"` `date +%Y/%m/%d --date="-192hour"` `date +%Y/%m/%d --date="-168hour"` \ 
+        `date +%Y/%m/%d --date="-144hour"` `date +%Y/%m/%d --date="-120hour"` `date +%Y/%m/%d --date="-96hour"` \
+        `date +%Y/%m/%d --date="-72hour"` `date +%Y/%m/%d --date="-48hour"` `date +%Y/%m/%d --date="-24hour"` \
+        )
+
+numdaysok=0
+numdaysoklimit=3
+for date in ${dates[@]}
+do 
+   echo ""
+   echo ""
+   echo ""
+   echo $date
+
+   # some counters
+   numdiff=0
+   numok=0
+   numpb=0
+   
+   # check always only $numdaysoklimit days
+   if [ $numdaysok -gt $numdaysoklimit ]
+   then
+      echo "INFO more than "$numdaysoklimit" ok -> stop. "
+      continue
+   fi
+   
+   # get paths
+   date2=`echo $date | sed -e 's/\///g'`
+   lprawpath="/daq/raw/"$date
+   lprawpath2="/loc_data/raw/"$date
+   lpziprawpath="/loc_data/zipraw/"$date
+   localrawpath="/scratch/from_lapalma/raw/"$date
+   localrawpath2="/data00/fact-construction/raw/"$date
+   localrawpath3="/fact/raw/"$date
+   localfailpath="/archive/fact/fails/raw/"$date
+   wuerawpath="/fact/raw/"$date
+   
+   # get disk usage and number of files for directory
+   daq=( `ssh fact@161.72.93.131 "if [ -d $lprawpath ]; then ls $lprawpath/* | wc -l; du -s -b --apparent-size $lprawpath; else echo '-1 -1 -1'; fi"` )
+   data=( `ssh fact@161.72.93.131 "if [ -d $lprawpath2 ]; then ls $lprawpath2/* | wc -l; du -s -b --apparent-size $lprawpath2; else echo '-1 -1 -1'; fi"` )
+   if [ ${daq[0]} -eq -1 ] && [ ${data[0]} -eq -1 ] 
+   then
+      echo "INFO no data available on daq for "$date
+      continue
+   fi
+   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
+   dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
+   data00=( `if [ -d $localrawpath2 ]; then ls $localrawpath2/* | wc -l; du -s -b --apparent-size $localrawpath2; else echo '-1 -1 -1'; fi` )
+   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* | wc -l; du -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
+   fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
+   wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
+   query="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM RunInfo WHERE fNight="$date2
+   querystart="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM "
+   queryjoin="LEFT JOIN RunInfo USING(fNight,fRunID) "
+   querywhere="WHERE fNight="$date2" AND NOT ISNULL(fStartTime) AND NOT ISNULL(fStopTime) AND ISNULL(fReturnCode)"
+   numruns=`sendquery`
+   if [ "$numruns" == "" ]
+   then
+      numruns=0
+   fi
+   query=$querystart"RawFileRsyncedISDCStatus "$queryjoin" "$querywhere
+   numrsynced=`sendquery`
+   if [ "$numrsynced" == "" ]
+   then
+      numrsynced=0
+   fi
+   query=$querystart"RawFileAvailISDCStatus "$queryjoin" "$querywhere
+   numisdc=`sendquery`
+   if [ "$numisdc" == "" ]
+   then
+      numisdc=0
+   fi
+   query=$querystart"RawFileAvailWueStatus "$queryjoin" "$querywhere
+   numwue=`sendquery`
+   if [ "$numwue" == "" ]
+   then
+      numwue=0
+   fi
+   echo "  daq     "${daq[@]}
+   echo "  data    "${data[@]}
+   echo "  zip     "${zip[@]}
+   if [ $date2 -gt 20120307 ]
+   then 
+      echo "  dl00    "${dl00[@]}
+   else
+      echo "  data00  "${data00[@]}
+   fi
+   echo "  arch    "${archive[@]}
+   echo "  fail    "${fails[@]}
+   echo "  wue     "${wue[@]}
+   echo "  runinfo "$numruns
+   echo "  rsynced "$numrsynced
+   echo "  isdc    "$numisdc
+   echo "  wue     "$numwue
+   
+   # check if file are available in the differen places
+   if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] 
+   then
+      echo "INFO data not available on /scratch on dl00 for "$date
+   fi
+   if [ ${data00[0]} -eq -1 ] && [ $date2 -gt 20120307 ]
+   then
+      echo "INFO data not available on /data00 for "$date
+   fi
+   if [ ${archive[0]} -eq -1 ] 
+   then
+      echo "INFO data not in archive for "$date
+   fi
+   
+   # check if number of files agree in the different places
+   # archive
+   if ! [ ${archive[0]} -eq -1 ]  && ! [ ${archive[0]} -eq ${daq[0]} ]
+   then
+      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
+      #check /archive/rev_1/failed
+      if ! [ ${fails[0]} -eq -1 ]
+      then
+         sum=`echo " ${fails[0]} + ${archive[0]} " | bc -l `
+         if ! [ $sum -eq ${daq[0]} ]
+         then
+            echo "ERROR number of files in archive ("$sum") wrong."
+            numpb=`echo " $numpb + 1 " | bc -l `
+            result="1"
+         else
+            numok=`echo " $numok + 1 " | bc -l `
+            result="0"
+         fi
+      fi
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result="0"
+   fi
+   # lp
+   if ! [ ${data[0]} -eq -1 ]  && ! [ ${data[0]} -eq ${daq[0]} ]
+   then
+      echo "WARN number of files on data (" ${data[0]}") does not agree with number of files on daq (" ${daq[0]}") for "$date
+      numpb=`echo " $numpb + 1 " | bc -l `
+      result=$result"1"
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   # dl00
+   if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${daq[0]} ]
+   then
+      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
+      numpb=`echo " $numpb + 1 " | bc -l `
+      result=$result"1"
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   # /data00 (to be removed later)
+   if ! [ ${data00[0]} -eq -1 ]  && ! [ ${data00[0]} -eq ${daq[0]} ]
+   then
+      echo "WARN number of files in /data00 (" ${data00[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
+   fi
+   # wue
+   if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${daq[0]} ]
+   then
+      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
+      numpb=`echo " $numpb + 1 " | bc -l `
+      result=$result"1"
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   
+   result=$result"-"
+   # check du for raw files 
+   # la palma
+   if ! [ ${daq[1]} -eq ${data[1]} ]
+   then
+      echo "WARN size of data doesn't agree on daq ("${daq[1]}") and data ("${data[1]}") for "$date
+      numdiff=`echo " $numdiff + 1 " | bc -l `
+      numpb=`echo " $numpb + 1 " | bc -l `
+      result=$result"1"
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   # check du for zipped raw files 
+   # dl00
+   if ! [ ${zip[1]} -eq ${dl00[1]} ] && [ $date2 -gt 20120307 ]
+   then
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
+      if ! [ ${dl00[1]} -eq -1 ]
+      then 
+         numdiff=`echo " $numdiff + 1 " | bc -l `
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   # data00 (to be removed later)
+   if ! [ ${zip[1]} -eq ${data00[1]} ] && [ $date2 -lt 20120308 ]
+   then
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and /data00 ("${data00[1]}") for "$date
+      if ! [ ${data00[1]} -eq -1 ]
+      then 
+         numdiff=`echo " $numdiff + 1 " | bc -l `
+      fi
+   fi
+   # archive
+   if ! [ ${zip[1]} -eq ${archive[1]} ]
+   then
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
+      if ! [ ${archive[1]} -eq -1 ]
+      then 
+         numdiff=`echo " $numdiff + 1 " | bc -l `
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   # wue
+   if ! [ ${zip[1]} -eq ${wue[1]} ]
+   then
+      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
+      if ! [ ${wue[1]} -eq -1 ]
+      then 
+         numdiff=`echo " $numdiff + 1 " | bc -l `
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+   else
+      numok=`echo " $numok + 1 " | bc -l `
+      result=$result"0"
+   fi
+   result=$result"-"
+
+   # check DB (only starting from 8.3.2012) (if-clause to be removed later)
+   if [ $date2 -gt 20120307 ]
+   then
+      # lp
+      if ! [ $numruns -eq ${daq[0]} ]
+      then
+         echo "WARN number of runs on daq ("${daq[0]}") not equal to number of runs ("$numruns")"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+      # dl00
+      if ! [ $numruns -eq $numrsynced ]
+      then
+         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+      # archive
+      if ! [ $numruns -eq $numisdc ]
+      then
+         echo "WARN number of rsynced in archive ("$numrsynced") not equal to number of runs ("$numruns")"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+      # wue
+      if ! [ $numruns -eq $numwue ]
+      then
+         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         result=$result"1"
+      else
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+      fi
+   fi
+   
+   echo "numdiff: "$numdiff
+   echo "INFO numok: "$numok
+   echo "INFO numpb: "$numpb
+   if [ $numdiff -gt 0 ]
+   then
+      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2
+      runs=`sendquery`
+      archivediffcounter=0
+      archiveokcounter=0
+      wuediffcounter=0
+      wueokcounter=0
+      dl00diffcounter=0
+      dl00okcounter=0
+      datadiffcounter=0
+      dataokcounter=0
+      for run in ${runs[@]}
+      do 
+         rawfile=$date2"_"`printf %03d $run`".fits"
+         rawfile2=$rawfile".gz"
+         
+         # get file sizes for run
+         sizedaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
+         sizedata=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
+         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
+         if ! [ ${dl00[1]} -eq -1 ]
+         then
+            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
+         fi
+         if ! [ ${data00[1]} -eq -1 ]
+         then
+            sizedata00=( `ls -l $localrawpath2/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
+         fi
+         if ! [ ${archive[1]} -eq -1 ]
+         then
+            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
+         fi
+         if ! [ ${wue[1]} -eq -1 ]
+         then
+            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
+         fi
+         
+         # check file sizes for run
+         # lp
+         if ! [ "$sizedaq" = "$sizedata" ]
+         then
+            echo "  "$rawfile" daq("$sizedaq") data("$sizedata")"
+            datadiffcounter=`echo " $datadiffcounter + 1 " | bc -l `
+         else
+            dataokcounter=`echo " $dataokcounter + 1 " | bc -l `
+         fi
+         # dl00
+         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
+            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
+         else
+            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
+         fi
+         # data00
+         if ! [ "$sizezip" = "$sizedata00" ] && ! [ ${data00[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") data00("$sizedata00")"
+         fi
+         # archive
+         if ! [ "$sizezip" = "$sizearchive" ] && ! [ ${archive[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive")"
+            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
+         else
+            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
+         fi
+         # wue
+         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
+            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
+         else
+            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
+         fi
+      done
+      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
+      drsruns=`sendquery`
+      for drsrun in ${drsruns[@]}
+      do 
+         rawfile=$date2"_"`printf %03d $run`"drs.fits"
+         rawfile2=$rawfile".gz"
+         # get file sizes for run
+         sizedaq=( `ssh fact@161.72.93.131 "ls -l ${lprawpath}/${rawfile} 2>/dev/null | awk '{ print \\\$5 }'"` )
+         sizedata=( `ssh fact@161.72.93.131 "ls -l $lprawpath2/$rawfile 2>/dev/null | awk '{ print \\\$5 }'"` )
+         sizezip=( `ssh fact@161.72.93.131 "ls -l $lpziprawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
+         if ! [ ${dl00[1]} -eq -1 ]
+         then
+            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
+         fi
+         if ! [ ${data00[1]} -eq -1 ]
+         then
+            sizedata00=( `ls -l $localrawpath2/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
+         fi
+         if ! [ ${archive[1]} -eq -1 ]
+         then
+            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
+         fi
+         if ! [ ${wue[1]} -eq -1 ]
+         then
+            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
+         fi
+         
+         # check file sizes for run
+         # lp
+         if ! [ "$sizedaq" = "$sizedata" ]
+         then
+            echo "  "$rawfile" daq("$sizedaq") data("$sizedata")"
+            datadiffcounter=`echo " $datadiffcounter + 1 " | bc -l `
+         else
+            dataokcounter=`echo " $dataokcounter + 1 " | bc -l `
+         fi
+         # dl00
+         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
+            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
+         else
+            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
+         fi
+         # data00
+         if ! [ "$sizezip" = "$sizedata00" ] && ! [ ${data00[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") data00("$sizedata00")"
+         fi
+         # archive
+         if ! [ "$sizezip" = "$sizearchive" ] && ! [ ${archive[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive")"
+            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
+         else
+            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
+         fi
+         # wue
+         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
+         then
+            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
+            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
+         else
+            wueokcounter=`echo " $wueokcounter + 1 " | bc -l `
+         fi
+      done
+      
+      result=$result"-"
+      echo "Wue:  ok: "$wueokcounter" diff: "$wuediffcounter
+      echo "Arch: ok: "$archiveokcounter" diff: "$archivediffcounter
+      # raw files
+      if [ $dataokcounter -eq ${data[0]} ]
+      then
+         numok=`echo " $numok + 1 " | bc -l `
+         result=$result"0"
+         echo "INFO "$dataokcounter" files are ok on data (raw)."
+      else
+         result=$result"1"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         echo "WARN "$datadiffcounter" files have a different size on data (raw)."
+      fi
+      # zipped files
+      # dl00
+      if [ $dl00okcounter -eq ${dl00[0]} ]
+      then
+         result=$result"0"
+         numok=`echo " $numok + 1 " | bc -l `
+         echo "INFO "$dl00okcounter" files are ok on dl00."
+      else
+         result=$result"1"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         echo "WARN "$dl00diffcounter" files have a different size on dl00."
+      fi
+      # archive
+      if [ $archiveokcounter -eq ${daq[0]} ]
+      then
+         result=$result"0"
+         numok=`echo " $numok + 1 " | bc -l `
+         echo "INFO "$archiveokcounter" files are ok in the archive."
+      else
+         result=$result"1"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         echo "WARN "$archivediffcounter" files have a different size in the archive."
+      fi
+      # wue
+      if [ $wueokcounter -eq ${wue[0]} ]
+      then
+         result=$result"0"
+         numok=`echo " $numok + 1 " | bc -l `
+         echo "INFO "$wueokcounter" files are ok in Wue."
+      else
+         result=$result"1"
+         numpb=`echo " $numpb + 1 " | bc -l `
+         echo "WARN "$wuediffcounter" files have a different size in Wue."
+      fi
+   fi
+   
+   # print summary: 
+   echo "INFO day ok: "$numdaysok
+   echo "INFO numok: "$numok
+   echo "INFO numpb: "$numpb
+   echo "result: (#files-dudir-db-filesize)"
+   echo "ldaw-ldaw-ldaw-ldaw"
+   echo $result
+   #if [ $numpb -eq 0 ]
+   if [ $numpb -lt 4 ]
+   then 
+      numdaysok=`echo " $numdaysok + 1 " | bc -l `
+   fi
+done
+
+printprocesslog "INFO finished $0"
+