Changeset 16678 for trunk


Ignore:
Timestamp:
06/03/13 22:40:40 (11 years ago)
Author:
Daniela Dorner
Message:
reworked output, added summary
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Transfer/CheckTransfer.sh

    r15599 r16678  
    77printprocesslog "INFO starting $0"
    88
    9 source /home_nfs/isdc/fact_opr/myagent.sh
     9logfile2=$logpath"/transfer/CheckTransfer.log"
     10date > $logfile2 2>&1
     11
     12diskusage=( `ssh fact@161.72.93.131 "df -P /daq" | grep daq ` )
     13# check if more than X GB are left on /daq
     14if [ ${diskusage[3]} -lt 700000 ]
     15then
     16   printprocesslog "WARN less than 700 GB left on /daq "
     17   echo "WARN less than 700 GB left on /daq "
     18   echo "WARN less than 700 GB left on /daq " >> $logfile2 2>&1
     19   sendemail="yes"
     20fi
     21
     22diskusage2=( `df -P /scratch | grep scratch ` )
     23# check if more than X GB are left on /scratch
     24if [ ${diskusage2[3]} -lt 500000 ]
     25then
     26   printprocesslog "WARN less than 500 GB left on /scratch "
     27   echo "WARN less than 500 GB left on /scratch "
     28   echo "WARN less than 500 GB left on /scratch " >> $logfile2 2>&1
     29   sendemail="yes"
     30fi
     31
     32# needed for transfer to phido
     33#source /home_nfs/isdc/fact_opr/myagent.sh
    1034
    1135# check first the disk in LP and on dl00
    … …  
    1337df -h /scratch
    1438echo ""
     39echo "" >> $logfile2 2>&1
    1540
    1641# check next the DB to know if some transfer processes failed or crashed
    … …  
    2651      #   echo $run
    2752      #done
     53      sendemail="yes"
    2854      echo -e "\e[1;31m\x1b[5m ==>\e[00m "$1": "${runs[@]}"\e[1;31m\x1b[5m <==\e[00m "
    29       echo "SELECT fNight, fRunId FROM "$1" "$where";"
     55      echo "SELECT fNight, fRunId, fStartTime, fStopTime, fReturnCode FROM "$1" "$where";"
    3056      echo "UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";"
     57      echo -e " ==> "$1": "${runs[@]}" <== " >> $logfile2 2>&1
     58      echo "---> Please check the DB and reset the processes if needed. " >> $logfile2 2>&1
     59      echo "to check: SELECT fNight, fRunId, fStartTime, fStopTime, fReturnCode FROM "$1" "$where";" >> $logfile2 2>&1
     60      echo "to reset: UPDATE "$1" SET fStartTime=NULL, fStopTime=NULL, fAvailable=NULL, fReturnCode=NULL, fProcessingSiteKey=NULL "$where";" >> $logfile2 2>&1
    3161   fi
    3262}
    … …  
    4171# RawFileAvailISDC needs a different treatment
    4272#  as return code 0 means that file is in fails folder in archive
    43 where="WHERE fReturnCode>0 OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) "
     73where="WHERE fReturnCode>0 OR (NOT ISNULL(fStartTime) AND ISNULL(fStopTime)) AND fStartTime < DATE_ADD(Now(), INTERVAL -1 HOUR) "
    4474check_runs_in_db "RawFileAvailISDCStatus"
    4575
    … …  
    5585dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` )
    5686
    57 #dates=( "2013/04/18" )
     87dates=( "2013/05/23" )
    5888#short="yes"
    5989
    … …  
    6494for date in ${dates[@]}
    6595do
    66    echo ""
    67    echo ""
    68    echo ""
    69    echo $date
     96   logfile=$logpath"/transfer/CheckTransfer_"`echo $date | sed -e 's/\//-/g'`".log"
     97   #echo ""
     98   #echo ""
     99   #echo ""
     100   #echo $date
     101   #echo "" > $logfile 2>&1
     102   #echo "" >> $logfile 2>&1
     103   #echo "" >> $logfile 2>&1
     104   echo $date > $logfile 2>&1
    70105
    71106   # some counters
    … …  
    79114   if [ $numdaysok -ge $numdaysoklimit ] && [ $(echo " $sumdata > $checklimit " | bc -l) -eq 1 ]
    80115   then
    81       echo "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
     116      printprocesslog "INFO more than "$numdaysoklimit" ok and more than "$checklimit" GB checked. "
    82117      continue
    83118   fi
    … …  
    102137   if [ ${newdaq[0]} -eq -1 ] && [ ${daq[0]} -eq -1 ]
    103138   then
    104       echo "INFO no data available on newdaq for "$date
     139      printprocesslog "INFO no data available on newdaq for "$date
     140      echo "INFO no data available on newdaq for "$date >> $logfile 2>&1
    105141      continue
    106142   fi
    107    zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
     143   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* 2>/dev/null | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
    108144   dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
    109    archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* | wc -l; du -L -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
     145   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* 2>/dev/null | wc -l; du -L -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
    110146   fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -L -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
    111147   wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` )
    … …  
    144180   #   numphido=0
    145181   #fi
    146    echo "disk: "
    147    echo "  newdaq  "${newdaq[@]}
    148    echo "  daq     "${daq[@]}
    149    echo "  data    "${zip[@]}
    150    echo "  dl00    "${dl00[@]}
    151    echo "  wue     "${wue[@]}
    152    echo "  arch    "${archive[@]}
    153    echo "  fail    "${fails[@]}
    154    #echo "  phido   "${phido[@]}
    155    echo "db: "
    156    echo "  runinfo "$numruns
    157    echo "  rsynced "$numrsynced
    158    echo "  isdc    "$numisdc
    159    echo "  wue     "$numwue
    160    #echo "  phido   "$numphido
     182   
     183   printprocesslog "disk: "
     184   printprocesslog "  newdaq  "${newdaq[@]}
     185   printprocesslog "  daq     "${daq[@]}
     186   printprocesslog "  data    "${zip[@]}
     187   printprocesslog "  dl00    "${dl00[@]}
     188   printprocesslog "  wue     "${wue[@]}
     189   printprocesslog "  arch    "${archive[@]}
     190   printprocesslog "  fail    "${fails[@]}
     191   #printprocesslog "  phido   "${phido[@]}
     192   printprocesslog "db: "
     193   printprocesslog "  runinfo "$numruns
     194   printprocesslog "  rsynced "$numrsynced
     195   printprocesslog "  isdc    "$numisdc
     196   printprocesslog "  wue     "$numwue
     197   #printprocesslog "  phido   "$numphido
     198   echo "disk: " >> $logfile 2>&1
     199   echo "  newdaq  "${newdaq[@]} >> $logfile 2>&1
     200   echo "  daq     "${daq[@]} >> $logfile 2>&1
     201   echo "  data    "${zip[@]} >> $logfile 2>&1
     202   echo "  dl00    "${dl00[@]} >> $logfile 2>&1
     203   echo "  wue     "${wue[@]} >> $logfile 2>&1
     204   echo "  arch    "${archive[@]} >> $logfile 2>&1
     205   echo "  fail    "${fails[@]} >> $logfile 2>&1
     206   #echo "  phido   "${phido[@]} >> $logfile 2>&1
     207   echo "db: " >> $logfile 2>&1
     208   echo "  runinfo "$numruns >> $logfile 2>&1
     209   echo "  rsynced "$numrsynced >> $logfile 2>&1
     210   echo "  isdc    "$numisdc >> $logfile 2>&1
     211   echo "  wue     "$numwue >> $logfile 2>&1
     212   #echo "  phido   "$numphido >> $logfile 2>&1
    161213   
    162214   # check if file are available in the different places
    163215   if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ]
    164216   then
    165       echo "INFO data not available on /scratch on dl00 for "$date
     217      printprocesslog "INFO data not available on /scratch on dl00 for "$date
     218      echo "INFO data not available on /scratch on dl00 for "$date >> $logfile 2>&1
    166219   fi
    167220   if [ ${archive[0]} -eq -1 ]
    168221   then
    169       echo "INFO data not in archive for "$date
     222      printprocesslog "INFO data not in archive for "$date
     223      echo "INFO data not in archive for "$date >> $logfile 2>&1
    170224   fi
    171225   
    … …  
    174228   if ! [ ${daq[0]} -eq -1 ]  && ! [ ${daq[0]} -eq ${newdaq[0]} ]
    175229   then
    176       echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date
     230      printprocesslog "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date
     231      echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date >> $logfile 2>&1
    177232      numpb=`echo " $numpb + 1 " | bc -l `
    178       result="1"
     233      result1="1"
    179234   else
    180235      numok=`echo " $numok + 1 " | bc -l `
    181       result="0"
     236      result1="0"
    182237   fi
    183238   # dl00
    184    if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${newdaq[0]} ]
    185    then
    186       echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     239   #if ! [ ${dl00[0]} -eq -1 ]  && ! [ ${dl00[0]} -eq ${newdaq[0]} ]
     240   if ! [ ${dl00[0]} -eq ${newdaq[0]} ]
     241   then
     242      printprocesslog "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     243      echo "WARN number of files on dl00 (" ${dl00[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
    187244      numpb=`echo " $numpb + 1 " | bc -l `
    188       result=$result"1"
     245      result1=$result1"1"
    189246   else
    190247      numok=`echo " $numok + 1 " | bc -l `
    191       result=$result"0"
     248      result1=$result1"0"
    192249   fi
    193250   # archive
    194251   if ! [ ${archive[0]} -eq -1 ]  && ! [ ${archive[0]} -eq ${newdaq[0]} ]
    195252   then
    196       echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     253      printprocesslog "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     254      echo "WARN number of files in archive (" ${archive[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
    197255      #check /archive/rev_1/failed
    198256      if ! [ ${fails[0]} -eq -1 ]
    … …  
    201259         if ! [ $sum -eq ${newdaq[0]} ]
    202260         then
    203             echo "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")."
     261            printprocesslog "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")."
     262            echo "ERROR number of files in whole archive ("$sum") is different from number of files in La Palma ("${newdaq[0]}")." >> $logfile 2>&1
    204263            numpb=`echo " $numpb + 1 " | bc -l `
    205             result=$result"1"
     264            result1=$result1"1"
    206265         else
    207266            numok=`echo " $numok + 1 " | bc -l `
    208             result=$result"0"
    209          fi
    210       else
    211          result=$result"1"
     267            result1=$result1"0"
     268         fi
     269      else
     270         result1=$result1"1"
    212271      fi
    213272   else
    … …  
    215274      then
    216275         numpb=`echo " $numpb + 1 " | bc -l `
    217          result=$result"1"
    218       else
    219          numok=`echo " $numok + 1 " | bc -l `
    220          result=$result"0"
     276         result1=$result1"1"
     277      else
     278         numok=`echo " $numok + 1 " | bc -l `
     279         result1=$result1"0"
    221280      fi
    222281   fi
    223282   # wue
    224    if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${newdaq[0]} ]
    225    then
    226       echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     283   #if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${newdaq[0]} ]
     284   if ! [ ${wue[0]} -eq ${newdaq[0]} ]
     285   then
     286      printprocesslog "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     287      echo "WARN number of files in Wue (" ${wue[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
    227288      numpb=`echo " $numpb + 1 " | bc -l `
    228       result=$result"1"
     289      result1=$result1"1"
    229290   else
    230291      numok=`echo " $numok + 1 " | bc -l `
    231       result=$result"0"
     292      result1=$result1"0"
    232293   fi
    233294   ## phido
    234295   #if ! [ ${phido[0]} -eq -1 ]  && ! [ ${phido[0]} -eq ${newdaq[0]} ]
    235296   #then
    236    #   echo "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     297   #   printprocesslog "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date
     298   #   echo "WARN number of files on Phido (" ${phido[0]}") does not agree with number of files in LP (" ${newdaq[0]}") for "$date >> $logfile 2>&1
    237299   #   numpb=`echo " $numpb + 1 " | bc -l `
    238    #   result=$result"1"
     300   #   result1=$result1"1"
    239301   #else
    240302   #   numok=`echo " $numok + 1 " | bc -l `
    241    #   result=$result"0"
     303   #   result1=$result1"0"
    242304   #fi
    243305   
    244    if [ "$short" = "yes" ] || [ "$result" != "0000" ]
    245    then
    246       echo $result
     306   if [ "$short" = "yes" ] || [ "$result1" != "0000" ]
     307   then
     308      printprocesslog "number of files does not yet agree in all sites ("$result1") -> do no further checking."
     309      echo "number of files does not yet agree in all sites ("$result1") -> do no further checking." >> $logfile 2>&1
     310      echo ""
     311      echo "SUMMARY for "$date
     312      echo "-----------------------"
     313      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) "
     314      echo "  "$date" is not yet transfered completely. For more details, please check the logfile "$logfile
     315      echo "" >> $logfile 2>&1
     316      echo "SUMMARY for "$date >> $logfile 2>&1
     317      echo "-----------------------" >> $logfile 2>&1
     318      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) " >> $logfile 2>&1
     319      echo "  "$date" is not yet transfered completely. For more details, please check the logfile "$logfile >> $logfile 2>&1
     320      echo "" >> $logfile2 2>&1
     321      echo "SUMMARY for "$date >> $logfile2 2>&1
     322      echo "-----------------------" >> $logfile2 2>&1
     323      echo "  number of files does not yet agree in all sites: "${newdaq[0]}" (newdaq) "${daq[0]}" (daq) "${zip[0]}" (data) "${dl00[0]}" (dl) "${wue[0]}" (wue) "${archive[0]}" (arch) "${fails[0]}" (fails) " >> $logfile2 2>&1
     324      echo "  "$date" is not yet transfered completely. For more details, please check the logfile "$logfile >> $logfile2 2>&1
     325      #echo "number of files does not yet agree in all sites ("$result1") -> do no further checking. "
     326      #echo "disk: "
     327      #echo "  newdaq  "${newdaq[@]}
     328      #echo "  daq     "${daq[@]}
     329      #echo "  data    "${zip[@]}
     330      #echo "  dl00    "${dl00[@]}
     331      #echo "  wue     "${wue[@]}
     332      #echo "  arch    "${archive[@]}
     333      #echo "  fail    "${fails[@]}
     334      ##echo "  phido   "${phido[@]}
     335      #echo "db: "
     336      #echo "  runinfo "$numruns
     337      #echo "  rsynced "$numrsynced
     338      #echo "  isdc    "$numisdc
     339      #echo "  wue     "$numwue
     340      ##echo "  phido   "$numphido
    247341      continue
    248342   fi
    249343   
    250    result=$result"-"
     344   #result=$result"-"
    251345   # check du for raw files
    252346   # la palma
    253347   if ! [ ${newdaq[1]} -eq ${daq[1]} ]
    254348   then
    255       echo "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date
     349      printprocesslog "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date
     350      echo "WARN size of data doesn't agree on newdaq ("${newdaq[1]}") and daq ("${daq[1]}") for "$date >> $logfile 2>&1
    256351      numdiff=`echo " $numdiff + 1 " | bc -l `
    257352      numpb=`echo " $numpb + 1 " | bc -l `
    258       result=$result"1"
     353      result2="1"
    259354   else
    260355      numok=`echo " $numok + 1 " | bc -l `
    261       result=$result"0"
     356      result2="0"
    262357   fi
    263358   # check du for zipped raw files
    … …  
    265360   if ! [ ${zip[1]} -eq ${dl00[1]} ] && [ $date2 -gt 20120307 ]
    266361   then
    267       echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
     362      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date
     363      echo "WARN size of data doesn't agree on data ("${zip[1]}") and dl00 ("${dl00[1]}") for "$date >> $logfile 2>&1
    268364      if ! [ ${dl00[1]} -eq -1 ]
    269365      then
    270366         numdiff=`echo " $numdiff + 1 " | bc -l `
    271367         numpb=`echo " $numpb + 1 " | bc -l `
    272          result=$result"1"
    273       else
    274          numok=`echo " $numok + 1 " | bc -l `
    275          result=$result"0"
     368         result2=$result2"1"
     369      else
     370         numok=`echo " $numok + 1 " | bc -l `
     371         result2=$result2"0"
    276372      fi
    277373   else
    278374      numok=`echo " $numok + 1 " | bc -l `
    279       result=$result"0"
     375      result2=$result2"0"
    280376   fi
    281377   # archive
    282378   if ! [ ${zip[1]} -eq ${archive[1]} ]
    283379   then
    284       echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
     380      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date
     381      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in archive ("${archive[1]}") for "$date >> $logfile 2>&1
    285382      if ! [ ${archive[1]} -eq -1 ]
    286383      then
    287384         numdiff=`echo " $numdiff + 1 " | bc -l `
    288385         numpb=`echo " $numpb + 1 " | bc -l `
    289          result=$result"1"
    290       else
    291          numok=`echo " $numok + 1 " | bc -l `
    292          result=$result"0"
     386         result2=$result2"1"
     387      else
     388         numok=`echo " $numok + 1 " | bc -l `
     389         result2=$result2"0"
    293390      fi
    294391   else
    295392      numok=`echo " $numok + 1 " | bc -l `
    296       result=$result"0"
     393      result2=$result2"0"
    297394   fi
    298395   # wue
    299396   if ! [ ${zip[1]} -eq ${wue[1]} ]
    300397   then
    301       echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
     398      printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date
     399      echo "WARN size of data doesn't agree on data ("${zip[1]}") and in Wue ("${wue[1]}") for "$date >> $logfile 2>&1
    302400      if ! [ ${wue[1]} -eq -1 ]
    303401      then
    304402         numdiff=`echo " $numdiff + 1 " | bc -l `
    305403         numpb=`echo " $numpb + 1 " | bc -l `
    306          result=$result"1"
    307       else
    308          numok=`echo " $numok + 1 " | bc -l `
    309          result=$result"0"
     404         result2=$result2"1"
     405      else
     406         numok=`echo " $numok + 1 " | bc -l `
     407         result2=$result2"0"
    310408      fi
    311409   else
    312410      numok=`echo " $numok + 1 " | bc -l `
    313       result=$result"0"
     411      result2=$result2"0"
    314412   fi
    315413   ## phido
    316414   #if ! [ ${zip[1]} -eq ${phido[1]} ]
    317415   #then
    318    #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date
     416   #   printprocesslog "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date
     417   #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and on Phido ("${phido[1]}") for "$date >> $logfile 2>&1
    319418   #   if ! [ ${phido[1]} -eq -1 ]
    320419   #   then
    321420   #      numdiff=`echo " $numdiff + 1 " | bc -l `
    322421   #      numpb=`echo " $numpb + 1 " | bc -l `
    323    #      result=$result"1"
     422   #      result2=$result2"1"
    324423   #   else
    325424   #      numok=`echo " $numok + 1 " | bc -l `
    326    #      result=$result"0"
     425   #      result2=$result2"0"
    327426   #   fi
    328427   #else
    329428   #   numok=`echo " $numok + 1 " | bc -l `
    330    #   result=$result"0"
     429   #   result2=$result2"0"
    331430   #fi
    332    result=$result"-"
     431   #result=$result"-"
    333432
    334433   # check DB (only starting from 8.3.2012) (if-clause to be removed later)
    … …  
    338437      if ! [ $numruns -eq ${newdaq[0]} ]
    339438      then
    340          echo "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")"
    341          numpb=`echo " $numpb + 1 " | bc -l `
    342          result=$result"1"
    343       else
    344          numok=`echo " $numok + 1 " | bc -l `
    345          result=$result"0"
     439         printprocesslog "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")"
     440         echo "WARN number of runs on newdaq ("${newdaq[0]}") not equal to number of runs ("$numruns")" >> $logfile 2>&1
     441         numpb=`echo " $numpb + 1 " | bc -l `
     442         result3="1"
     443      else
     444         numok=`echo " $numok + 1 " | bc -l `
     445         result3="0"
    346446      fi
    347447      # dl00
    348448      if ! [ $numruns -eq $numrsynced ]
    349449      then
    350          echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
    351          numpb=`echo " $numpb + 1 " | bc -l `
    352          result=$result"1"
    353       else
    354          numok=`echo " $numok + 1 " | bc -l `
    355          result=$result"0"
     450         printprocesslog "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")"
     451         echo "WARN number of rsynced runs ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
     452         numpb=`echo " $numpb + 1 " | bc -l `
     453         result3=$result3"1"
     454      else
     455         numok=`echo " $numok + 1 " | bc -l `
     456         result3=$result3"0"
    356457      fi
    357458      # archive
    358459      if ! [ $numruns -eq $numisdc ]
    359460      then
    360          echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
    361          numpb=`echo " $numpb + 1 " | bc -l `
    362          result=$result"1"
    363       else
    364          numok=`echo " $numok + 1 " | bc -l `
    365          result=$result"0"
     461         printprocesslog "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
     462         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")" >> $logfile 2>&1
     463         numpb=`echo " $numpb + 1 " | bc -l `
     464         result3=$result3"1"
     465      else
     466         numok=`echo " $numok + 1 " | bc -l `
     467         result3=$result3"0"
    366468      fi
    367469      # wue
    368470      if ! [ $numruns -eq $numwue ]
    369471      then
    370          echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
    371          numpb=`echo " $numpb + 1 " | bc -l `
    372          result=$result"1"
    373       else
    374          numok=`echo " $numok + 1 " | bc -l `
    375          result=$result"0"
     472         printprocesslog "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")"
     473         echo "WARN number of backuped in Wue ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
     474         numpb=`echo " $numpb + 1 " | bc -l `
     475         result3=$result3"1"
     476      else
     477         numok=`echo " $numok + 1 " | bc -l `
     478         result3=$result3"0"
    376479      fi
    377480      ## phido
    378481      #if ! [ $numruns -eq $numphido ]
    379482      #then
    380       #   echo "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")"
     483      #   printprocesslog "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")"
     484      #   echo "WARN number of backuped on Phido ("$numrsynced") not equal to number of runs ("$numruns")" >> $logfile 2>&1
    381485      #   numpb=`echo " $numpb + 1 " | bc -l `
    382       #   result=$result"1"
     486      #   result3=$result3"1"
    383487      #else
    384488      #   numok=`echo " $numok + 1 " | bc -l `
    385       #   result=$result"0"
     489      #   result3=$result3"0"
    386490      #fi
    387491   fi
    388492   
    389493   #numdiff=0 # add for debugging so that single file sizes are not checked
    390    echo "numdiff: "$numdiff
    391    echo "INFO numok: "$numok
    392    echo "INFO numpb: "$numpb
     494   printprocesslog "numdiff: "$numdiff
     495   printprocesslog "INFO numok: "$numok
     496   printprocesslog "INFO numpb: "$numpb
     497   echo "numdiff: "$numdiff >> $logfile 2>&1
     498   echo "INFO numok: "$numok >> $logfile 2>&1
     499   echo "INFO numpb: "$numpb >> $logfile 2>&1
    393500   #if [ $numdiff -gt 0 ]
    394501   if [ $numdiff -ge 0 ]
    … …  
    406513      daqdiffcounter=0
    407514      daqokcounter=0
    408       echo "INFO found "${#runs[@]}" rawfiles in DB."
     515      printprocesslog "INFO found "${#runs[@]}" rawfiles in DB."
     516      echo "INFO found "${#runs[@]}" rawfiles in DB." >> $logfile 2>&1
    409517      for run in ${runs[@]}
    410518      do
    … …  
    432540            sizewue=( `ssh operator@coma.astro.uni-wuerzburg.de "ls -l $wuerawpath/$rawfile2 2>/dev/null | awk '{ print \\\$5 }'"` )
    433541         fi
    434 #         echo "vgl "$run": "$sizezip" - "$sizearchive
    435542         #if ! [ ${phido[1]} -eq -1 ]
    436543         #then
    … …  
    442549         if ! [ "$sizenewdaq" = "$sizedaq" ]
    443550         then
    444             echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
     551            printprocesslog "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
     552            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")" >> $logfile 2>&1
    445553            daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
    446554         else
    … …  
    450558         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
    451559         then
    452             echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
     560            printprocesslog "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
     561            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")" >> $logfile 2>&1
    453562            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
    454563         else
    … …  
    460569         if [ ${archive[1]} -ne -1 -a "$sizezip" != "$sizearchive" -a "$sizezip" != "$sizefails" ]
    461570         then
    462             echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
    463             echo " "$sizezip"-"$sizearchive"-"${archive[1]}"-"$sizezip"-"$sizefails"-"${fails[1]}
     571            printprocesslog "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     572            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")" >> $logfile 2>&1
     573            #echo " "$sizezip"-"$sizearchive"-"${archive[1]}"-"$sizezip"-"$sizefails"-"${fails[1]}
    464574            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
    465575         else
    … …  
    469579         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
    470580         then
    471             echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
     581            printprocesslog "  "$rawfile2" data("$sizezip") wue("$sizewue")"
     582            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")" >> $logfile 2>&1
    472583            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
    473584         else
    … …  
    477588         #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
    478589         #then
    479          #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")"
     590         #   printprocesslog "  "$rawfile2" data("$sizezip") phido("$sizephido")"
     591         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")" >> $logfile 2>&1
    480592         #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
    481593         #else
    … …  
    485597      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
    486598      drsruns=( `sendquery` )
    487       echo "INFO found "${#drsruns[@]}" drsfiles in DB."
     599      printprocesslog "INFO found "${#drsruns[@]}" drsfiles in DB."
     600      echo "INFO found "${#drsruns[@]}" drsfiles in DB." >> $logfile 2>&1
    488601      for drsrun in ${drsruns[@]}
    489602      do
    … …  
    519632         if ! [ "$sizenewdaq" = "$sizedaq" ]
    520633         then
    521             echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
     634            printprocesslog "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")"
     635            echo "  "$rawfile" newdaq("$sizenewdaq") daq("$sizedaq")" >> $logfile 2>&1
    522636            daqdiffcounter=`echo " $daqdiffcounter + 1 " | bc -l `
    523637         else
    … …  
    527641         if ! [ "$sizezip" = "$sizedl00" ] && ! [ ${dl00[1]} -eq -1 ]
    528642         then
    529             echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
     643            printprocesslog "  "$rawfile2" data("$sizezip") dl00("$sizedl00")"
     644            echo "  "$rawfile2" data("$sizezip") dl00("$sizedl00")" >> $logfile 2>&1
    530645            dl00diffcounter=`echo " $dl00diffcounter + 1 " | bc -l `
    531646         else
    … …  
    535650         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 -a "$sizearchive" != "" ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1 -a "$sizefails" != "" ]
    536651         then
    537             echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     652            printprocesslog "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     653            echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")" >> $logfile 2>&1
    538654            archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
    539655         else
    … …  
    543659         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
    544660         then
    545             echo "  "$rawfile2" data("$sizezip") wue("$sizewue")"
     661            printprocesslog "  "$rawfile2" data("$sizezip") wue("$sizewue")"
     662            echo "  "$rawfile2" data("$sizezip") wue("$sizewue")" >> $logfile 2>&1
    546663            wuediffcounter=`echo " $wuediffcounter + 1 " | bc -l `
    547664         else
    … …  
    551668         #if ! [ "$sizezip" = "$sizephido" ] && ! [ ${phido[1]} -eq -1 ]
    552669         #then
    553          #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")"
     670         #   printprocesslog "  "$rawfile2" data("$sizezip") phido("$sizephido")"
     671         #   echo "  "$rawfile2" data("$sizezip") phido("$sizephido")" >> $logfile 2>&1
    554672         #   phidodiffcounter=`echo " $phidodiffcounter + 1 " | bc -l `
    555673         #else
    … …  
    558676      done
    559677     
    560       result=$result"-"
     678      #result=$result"-"
    561679      # raw files
    562680      if [ $daqokcounter -eq ${daq[0]} ]
    563681      then
    564682         numok=`echo " $numok + 1 " | bc -l `
    565          result=$result"0"
    566       else
    567          result=$result"1"
     683         result4="0"
     684      else
     685         result4="1"
    568686         numpb=`echo " $numpb + 1 " | bc -l `
    569687      fi
    … …  
    572690      if [ $dl00okcounter -eq ${dl00[0]} ]
    573691      then
    574          result=$result"0"
    575          numok=`echo " $numok + 1 " | bc -l `
    576       else
    577          result=$result"1"
     692         result4=$result4"0"
     693         numok=`echo " $numok + 1 " | bc -l `
     694      else
     695         result4=$result4"1"
    578696         numpb=`echo " $numpb + 1 " | bc -l `
    579697      fi
    … …  
    587705      if [ $archiveokcounter -eq ${daq[0]} ]
    588706      then
    589          result=$result"0"
    590          numok=`echo " $numok + 1 " | bc -l `
    591       else
    592          result=$result"1"
     707         result4=$result4"0"
     708         numok=`echo " $numok + 1 " | bc -l `
     709      else
     710         result4=$result4"1"
    593711         numpb=`echo " $numpb + 1 " | bc -l `
    594712      fi
    … …  
    596714      if [ $wueokcounter -eq ${wue[0]} ]
    597715      then
    598          result=$result"0"
    599          numok=`echo " $numok + 1 " | bc -l `
    600       else
    601          result=$result"1"
     716         result4=$result4"0"
     717         numok=`echo " $numok + 1 " | bc -l `
     718      else
     719         result4=$result4"1"
    602720         numpb=`echo " $numpb + 1 " | bc -l `
    603721      fi
    … …  
    605723      #if [ $phidookcounter -eq ${phido[0]} ]
    606724      #then
    607       #   result=$result"0"
     725      #   result4=$result4"0"
    608726      #   numok=`echo " $numok + 1 " | bc -l `
    609727      #else
    610       #   result=$result"1"
     728      #   result4=$result4"1"
    611729      #   numpb=`echo " $numpb + 1 " | bc -l `
    612730      #fi
    613       echo "INFO "$daqokcounter" files are ok on daq (raw)."
    614       echo "INFO "$dl00okcounter" files are ok on dl00."
    615       echo "INFO "$wueokcounter" files are ok in Wue."
    616       echo "INFO "$archiveokcounter" files are ok in the archive."
    617       #echo "INFO "$phidookcounter" files are ok on Phido."
    618       echo "WARN "$daqdiffcounter" files have a different size on daq (raw)."
    619       echo "WARN "$dl00diffcounter" files have a different size on dl00."
    620       echo "WARN "$wuediffcounter" files have a different size in Wue."
    621       echo "WARN "$archivediffcounter" files have a different size in the archive."
    622       #echo "WARN "$phidodiffcounter" files have a different size on Phido."
     731      printprocesslog "INFO "$daqokcounter" files are ok on daq (raw)."
     732      printprocesslog "INFO "$dl00okcounter" files are ok on dl00."
     733      printprocesslog "INFO "$wueokcounter" files are ok in Wue."
     734      printprocesslog "INFO "$archiveokcounter" files are ok in the archive."
     735      #printprocesslog "INFO "$phidookcounter" files are ok on Phido."
     736      printprocesslog "WARN "$daqdiffcounter" files have a different size on daq (raw)."
     737      printprocesslog "WARN "$dl00diffcounter" files have a different size on dl00."
     738      printprocesslog "WARN "$wuediffcounter" files have a different size in Wue."
     739      printprocesslog "WARN "$archivediffcounter" files have a different size in the archive."
     740      #printprocesslog "WARN "$phidodiffcounter" files have a different size on Phido."
     741      echo "INFO "$daqokcounter" files are ok on daq (raw)." >> $logfile 2>&1
     742      echo "INFO "$dl00okcounter" files are ok on dl00." >> $logfile 2>&1
     743      echo "INFO "$wueokcounter" files are ok in Wue." >> $logfile 2>&1
     744      echo "INFO "$archiveokcounter" files are ok in the archive." >> $logfile 2>&1
     745      #echo "INFO "$phidookcounter" files are ok on Phido." >> $logfile 2>&1
     746      echo "WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
     747      echo "WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
     748      echo "WARN "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
     749      echo "WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
     750      #echo "WARN "$phidodiffcounter" files have a different size on Phido." >> $logfile 2>&1
    623751   fi
    624752   
    625753   # print summary:
    626    echo "INFO day ok: "$numdaysok
    627    echo "INFO numok: "$numok
    628    echo "INFO numpb: "$numpb
    629    echo "result:"
    630    echo "(#files-dudir-db-filesize)"
    631    #echo " ldawp-ldawp-ldawp-ldawp"
    632    echo " ldaw-ldaw-ldaw-ldaw"
    633    echo " "$result
     754   printprocesslog "INFO day ok: "$numdaysok
     755   printprocesslog "INFO numok: "$numok
     756   printprocesslog "INFO numpb: "$numpb
     757   printprocesslog "result:"
     758   printprocesslog "(#files-dudir-db-filesize)"
     759   #printprocesslog " ldawp-ldawp-ldawp-ldawp"
     760   printprocesslog " ldaw-ldaw-ldaw-ldaw"
     761   printprocesslog " "$result1"-"$result2"-"$result3"-"$result4
     762   echo "INFO day ok: "$numdaysok >> $logfile 2>&1
     763   echo "INFO numok: "$numok >> $logfile 2>&1
     764   echo "INFO numpb: "$numpb >> $logfile 2>&1
     765   echo "result:" >> $logfile 2>&1
     766   echo "(#files-dudir-db-filesize)" >> $logfile 2>&1
     767   #echo " ldawp-ldawp-ldawp-ldawp" >> $logfile 2>&1
     768   echo " ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1
     769   echo " "$result1"-"$result2"-"$result3"-"$result4 >> $logfile 2>&1
    634770   sumdatanew=`echo " ( ${daq[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l | cut -d. -f1`
    635771   sumdata=`echo " $sumdata + $sumdatanew " | bc -l | cut -d. -f1`
    636    echo "checked alread "$sumdata" GB. "$sumdatanew" "${daq[1]}" "${zip[1]}
     772   printprocesslog "checked alread "$sumdata" GB. "$sumdatanew" "${daq[1]}" "${zip[1]}
    637773   if [ $numpb -lt 4 ]
    638774   then
    639775      numdaysok=`echo " $numdaysok + 1 " | bc -l `
    640776   fi
     777
     778   echo "SUMMARY for "$date
     779   echo "-----------------------"
     780   echo "" >> $logfile 2>&1
     781   echo "SUMMARY for "$date >> $logfile 2>&1
     782   echo "-----------------------" >> $logfile 2>&1
     783   echo "" >> $logfile2 2>&1
     784   echo "SUMMARY for "$date >> $logfile2 2>&1
     785   echo "-----------------------" >> $logfile2 2>&1
     786   #echo "res1:"$result1
     787   #echo "res3:"$result3
     788   #echo "res4:"$result4
     789   #echo "arch:"${archive[0]}
     790   #echo "isdc:"$numisdc
     791   if [ "$result1" = "0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ]
     792   then
     793      echo "  EVERYTHING is ok. "$date" can be deleted. "
     794      echo "  EVERYTHING is ok. "$date" can be deleted. " >> $logfile 2>&1
     795      echo "  EVERYTHING is ok. "$date" can be deleted. " >> $logfile2 2>&1
     796      echo "    Details in the logfile "$logfile
     797      echo "    Details in the logfile "$logfile >> $logfile 2>&1
     798      echo "    Details in the logfile "$logfile >> $logfile2 2>&1
     799   else
     800      if [ "$result1" = "0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ]
     801      then
     802         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. "
     803         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " >> $logfile 2>&1
     804         echo "  "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " >> $logfile2 2>&1
     805         echo "  TRANSFER is ok. "$date" can be deleted. "
     806         echo "  TRANSFER is ok. "$date" can be deleted. " >> $logfile 2>&1
     807         echo "  TRANSFER is ok. "$date" can be deleted. " >> $logfile2 2>&1
     808         echo "    Details in the logfile "$logfile
     809         echo "    Details in the logfile "$logfile >> $logfile 2>&1
     810         echo "    Details in the logfile "$logfile >> $logfile2 2>&1
     811      else
     812         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile
     813         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile 2>&1
     814         echo "  "$date" is not yet transfered completely. Please check the logfile "$logfile >> $logfile2 2>&1
     815         echo "    resetting jobs in the DB might be needed."
     816         echo "    resetting jobs in the DB might be needed." >> $logfile 2>&1
     817         echo "    resetting jobs in the DB might be needed." >> $logfile2 2>&1
     818      fi
     819   fi
     820   if [ $daqdiffcounter -gt 0 ]
     821   then
     822      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)."
     823      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile 2>&1
     824      echo "  WARN "$daqdiffcounter" files have a different size on daq (raw)." >> $logfile2 2>&1
     825   fi
     826   if [ $dl00diffcounter -gt 0 ]
     827   then
     828      echo "  WARN "$dl00diffcounter" files have a different size on dl00."
     829      echo "  WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile 2>&1
     830      echo "  WARN "$dl00diffcounter" files have a different size on dl00." >> $logfile2 2>&1
     831   fi
     832   if [ $wuediffcounter -gt 0 ]
     833   then
     834      echo "  WARN "$wuediffcounter" files have a different size in Wue."
     835      echo "  WARN "$wuediffcounter" files have a different size in Wue." >> $logfile 2>&1
     836      echo "  WARN "$wuediffcounter" files have a different size in Wue." >> $logfile2 2>&1
     837   fi
     838   if [ $archivediffcounter -gt 0 ]
     839   then
     840      echo "  WARN "$archivediffcounter" files have a different size in the archive."
     841      echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1
     842      echo "  WARN "$archivediffcounter" files have a different size in the archive." >> $logfile2 2>&1
     843   fi
    641844done
    642845
     846#sendemail="yes"
     847if [ "$sendemail" = "yes" ]
     848then
     849   echo "INFO send email with "$logfile2"to shift@fact-project.org "
     850   printprocesslog "INFO send email with "$logfile2"to shift@fact-project.org "
     851   cat $logfile2 | mail -s "testmail for info on deleting data" shift@fact-project.org
     852fi
     853
    643854printprocesslog "INFO finished $0"
    644855
Note: See TracChangeset for help on using the changeset viewer.