Changeset 13171 for trunk/DataCheck


Ignore:
Timestamp:
03/22/12 16:55:48 (13 years ago)
Author:
Daniela Dorner
Message:
improved output, fixed bugs
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Transfer/CheckTransfer.sh

    r13154 r13171  
    11#!/bin/bash
    22#
    3 # This script checks whether data can be transferred
     3# This script checks whether data can be deleted
    44#
    55
    … …  
    1414        `date +%Y/%m/%d --date="-72hour"` `date +%Y/%m/%d --date="-48hour"` `date +%Y/%m/%d --date="-24hour"` \
    1515        )
     16
     17dates=( "2012/03/12" )
    1618
    1719numdaysok=0
    … …  
    4244   lpziprawpath="/loc_data/zipraw/"$date
    4345   localrawpath="/scratch/from_lapalma/raw/"$date
    44    localrawpath2="/data00/fact-construction/raw/"$date
     46   #localrawpath2="/data00/fact-construction/raw/"$date
    4547   localrawpath3="/fact/raw/"$date
    4648   localfailpath="/archive/fact/fails/raw/"$date
    … …  
    5759   zip=( `ssh fact@161.72.93.131 "if [ -d $lpziprawpath ]; then ls $lpziprawpath/* | wc -l; du -s -b --apparent-size $lpziprawpath; else echo '-1 -1 -1'; fi"` )
    5860   dl00=( `if [ -d $localrawpath ]; then ls $localrawpath/* | wc -l; du -s -b --apparent-size $localrawpath; else echo '-1 -1 -1'; fi` )
    59    data00=( `if [ -d $localrawpath2 ]; then ls $localrawpath2/* | wc -l; du -s -b --apparent-size $localrawpath2; else echo '-1 -1 -1'; fi` )
     61   #data00=( `if [ -d $localrawpath2 ]; then ls $localrawpath2/* | wc -l; du -s -b --apparent-size $localrawpath2; else echo '-1 -1 -1'; fi` )
    6062   archive=( `if [ -d $localrawpath3 ]; then ls $localrawpath3/* | wc -l; du -s -b --apparent-size $localrawpath3; else echo '-1 -1 -1'; fi` )
    6163   fails=( `if [ -d $localfailpath ]; then ls $localfailpath/* | wc -l; du -s -b --apparent-size $localfailpath; else echo '-1 -1 -1'; fi` )
    … …  
    9193   echo "  data    "${data[@]}
    9294   echo "  zip     "${zip[@]}
    93    if [ $date2 -gt 20120307 ]
    94    then
     95   #if [ $date2 -gt 20120307 ]
     96   #then
    9597      echo "  dl00    "${dl00[@]}
    96    else
    97       echo "  data00  "${data00[@]}
    98    fi
     98   #else
     99   #   echo "  data00  "${data00[@]}
     100   #fi
    99101   echo "  arch    "${archive[@]}
    100102   echo "  fail    "${fails[@]}
    … …  
    105107   echo "  wue     "$numwue
    106108   
    107    # check if file are available in the differen places
     109   # check if file are available in the different places
    108110   if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ]
    109111   then
    110112      echo "INFO data not available on /scratch on dl00 for "$date
    111113   fi
    112    if [ ${data00[0]} -eq -1 ] && [ $date2 -gt 20120307 ]
    113    then
    114       echo "INFO data not available on /data00 for "$date
    115    fi
     114   #if [ ${data00[0]} -eq -1 ] && [ $date2 -gt 20120307 ]
     115   #then
     116   #   echo "INFO data not available on /data00 for "$date
     117   #fi
    116118   if [ ${archive[0]} -eq -1 ]
    117119   then
    … …  
    130132         if ! [ $sum -eq ${daq[0]} ]
    131133         then
    132             echo "ERROR number of files in archive ("$sum") wrong."
     134            echo "ERROR number of files in archive ("$sum") is different from number of files in La Palma ("${daq[0]}")."
    133135            numpb=`echo " $numpb + 1 " | bc -l `
    134136            result="1"
    … …  
    137139            result="0"
    138140         fi
     141      else
     142         result="1"
    139143      fi
    140144   else
    … …  
    162166      result=$result"0"
    163167   fi
    164    # /data00 (to be removed later)
    165    if ! [ ${data00[0]} -eq -1 ]  && ! [ ${data00[0]} -eq ${daq[0]} ]
    166    then
    167       echo "WARN number of files in /data00 (" ${data00[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
    168    fi
     168   ## /data00 (to be removed later)
     169   #if ! [ ${data00[0]} -eq -1 ]  && ! [ ${data00[0]} -eq ${daq[0]} ]
     170   #then
     171   #   echo "WARN number of files in /data00 (" ${data00[0]}") does not agree with number of files in LP (" ${daq[0]}") for "$date
     172   #fi
    169173   # wue
    170174   if ! [ ${wue[0]} -eq -1 ]  && ! [ ${wue[0]} -eq ${daq[0]} ]
    … …  
    209213      result=$result"0"
    210214   fi
    211    # data00 (to be removed later)
    212    if ! [ ${zip[1]} -eq ${data00[1]} ] && [ $date2 -lt 20120308 ]
    213    then
    214       echo "WARN size of data doesn't agree on data ("${zip[1]}") and /data00 ("${data00[1]}") for "$date
    215       if ! [ ${data00[1]} -eq -1 ]
    216       then
    217          numdiff=`echo " $numdiff + 1 " | bc -l `
    218       fi
    219    fi
     215   ## data00 (to be removed later)
     216   #if ! [ ${zip[1]} -eq ${data00[1]} ] && [ $date2 -lt 20120308 ]
     217   #then
     218   #   echo "WARN size of data doesn't agree on data ("${zip[1]}") and /data00 ("${data00[1]}") for "$date
     219   #   if ! [ ${data00[1]} -eq -1 ]
     220   #   then
     221   #      numdiff=`echo " $numdiff + 1 " | bc -l `
     222   #   fi
     223   #fi
    220224   # archive
    221225   if ! [ ${zip[1]} -eq ${archive[1]} ]
    … …  
    280284      if ! [ $numruns -eq $numisdc ]
    281285      then
    282          echo "WARN number of rsynced in archive ("$numrsynced") not equal to number of runs ("$numruns")"
     286         echo "WARN number of ingested files in archive ("$numisdc") not equal to number of runs ("$numruns")"
    283287         numpb=`echo " $numpb + 1 " | bc -l `
    284288         result=$result"1"
    … …  
    305309   then
    306310      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2
    307       runs=`sendquery`
     311      runs=( `sendquery` )
    308312      archivediffcounter=0
    309313      archiveokcounter=0
    … …  
    314318      datadiffcounter=0
    315319      dataokcounter=0
     320      echo "INFO found "${#runs[@]}" rawfiles in DB."
    316321      for run in ${runs[@]}
    317322      do
    … …  
    327332            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
    328333         fi
    329          if ! [ ${data00[1]} -eq -1 ]
    330          then
    331             sizedata00=( `ls -l $localrawpath2/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
    332          fi
     334         #if ! [ ${data00[1]} -eq -1 ]
     335         #then
     336         #   sizedata00=( `ls -l $localrawpath2/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
     337         #fi
    333338         if ! [ ${archive[1]} -eq -1 ]
    334339         then
    335340            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
     341         fi
     342         if ! [ ${fails[1]} -eq -1 ]
     343         then
     344            sizefails=( `ls -l $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
    336345         fi
    337346         if ! [ ${wue[1]} -eq -1 ]
    … …  
    357366            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
    358367         fi
    359          # data00
    360          if ! [ "$sizezip" = "$sizedata00" ] && ! [ ${data00[1]} -eq -1 ]
    361          then
    362             echo "  "$rawfile2" data("$sizezip") data00("$sizedata00")"
    363          fi
     368         ## data00
     369         #if ! [ "$sizezip" = "$sizedata00" ] && ! [ ${data00[1]} -eq -1 ]
     370         #then
     371         #   echo "  "$rawfile2" data("$sizezip") data00("$sizedata00")"
     372         #fi
    364373         # archive
    365          if ! [ "$sizezip" = "$sizearchive" ] && ! [ ${archive[1]} -eq -1 ]
    366          then
    367             echo "  "$rawfile2" data("$sizezip") archive("$sizearchive")"
    368             archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
     374         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1  ]
     375         #if ! [ "$sizezip" = "$sizearchive" ] && ! [ ${archive[1]} -eq -1 ]
     376         then
     377            #echo "  --"$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     378            #if ! [ "$sizezip" = "$sizefails" ] && ! [ ${fails[1]} -eq -1 ]
     379            #then
     380               echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     381               archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
     382            #else
     383            #   archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
     384            #fi
    369385         else
    370386            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
    371387         fi
     388         #if ! [ "$sizezip" = "" ] && ! [ "$sizearchive" = "" ] && ! [ ${archive[1]} -eq -1 ]
     389         #then
     390         #   if [ $sizezip -gt $sizearchive ]
     391         #   then
     392         #      echo "ERROR "$rawfile" smaller in archive ("$sizearchive") than on data ("$sizezip")"
     393         #   fi
     394         #fi
    372395         # wue
    373396         if ! [ "$sizezip" = "$sizewue" ] && ! [ ${wue[1]} -eq -1 ]
    … …  
    380403      done
    381404      query="SELECT fRunID FROM RunInfo WHERE fNight="$date2" AND fHasDrsFile=1"
    382       drsruns=`sendquery`
     405      drsruns=( `sendquery` )
     406      echo "INFO found "${#drsruns[@]}" drsfiles in DB."
    383407      for drsrun in ${drsruns[@]}
    384408      do
    … …  
    393417            sizedl00=( `ls -l $localrawpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
    394418         fi
    395          if ! [ ${data00[1]} -eq -1 ]
    396          then
    397             sizedata00=( `ls -l $localrawpath2/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
    398          fi
     419         #if ! [ ${data00[1]} -eq -1 ]
     420         #then
     421         #   sizedata00=( `ls -l $localrawpath2/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
     422         #fi
    399423         if ! [ ${archive[1]} -eq -1 ]
    400424         then
    401425            sizearchive=( `ls -l $localrawpath3/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
     426         fi
     427         if ! [ ${fails[1]} -eq -1 ]
     428         then
     429            sizefails=( `ls -l $localfailpath/$rawfile2 2>/dev/null | awk '{ print \$5 }'` )
    402430         fi
    403431         if ! [ ${wue[1]} -eq -1 ]
    … …  
    423451            dl00okcounter=`echo " $dl00okcounter + 1 " | bc -l `
    424452         fi
    425          # data00
    426          if ! [ "$sizezip" = "$sizedata00" ] && ! [ ${data00[1]} -eq -1 ]
    427          then
    428             echo "  "$rawfile2" data("$sizezip") data00("$sizedata00")"
    429          fi
     453         ## data00
     454         #if ! [ "$sizezip" = "$sizedata00" ] && ! [ ${data00[1]} -eq -1 ]
     455         #then
     456         #   echo "  "$rawfile2" data("$sizezip") data00("$sizedata00")"
     457         #fi
    430458         # archive
    431          if ! [ "$sizezip" = "$sizearchive" ] && ! [ ${archive[1]} -eq -1 ]
    432          then
    433             echo "  "$rawfile2" data("$sizezip") archive("$sizearchive")"
    434             archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
     459         #if [ ! [ "$sizezip" = "$sizearchive" ] && ! [ ${archive[1]} -eq -1 ] ] || [ ! [ "$sizezip" = "$sizefails" ] && ! [ ${fails[1]} -eq -1 ] ]
     460         if [  "$sizezip" != "$sizearchive"  -a  ${archive[1]} -ne -1 ] || [  "$sizezip" != "$sizefails" -a ${fails[1]} -ne -1  ]
     461         then
     462            #echo "  --"$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     463            #if
     464            #then
     465               echo "  "$rawfile2" data("$sizezip") archive("$sizearchive"/"$sizefails")"
     466               archivediffcounter=`echo " $archivediffcounter + 1 " | bc -l `
     467            #else
     468            #   archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
     469            #fi
    435470         else
    436471            archiveokcounter=`echo " $archiveokcounter + 1 " | bc -l `
    … …  
    447482     
    448483      result=$result"-"
    449       echo "Wue:  ok: "$wueokcounter" diff: "$wuediffcounter
    450       echo "Arch: ok: "$archiveokcounter" diff: "$archivediffcounter
     484      #echo "Wue:  ok: "$wueokcounter" diff: "$wuediffcounter
     485      #echo "Arch: ok: "$archiveokcounter" diff: "$archivediffcounter
    451486      # raw files
     487      echo "INFO "$dataokcounter" files are ok on data (raw)."
     488      echo "WARN "$datadiffcounter" files have a different size on data (raw)."
    452489      if [ $dataokcounter -eq ${data[0]} ]
    453490      then
    454491         numok=`echo " $numok + 1 " | bc -l `
    455492         result=$result"0"
    456          echo "INFO "$dataokcounter" files are ok on data (raw)."
    457       else
    458          result=$result"1"
    459          numpb=`echo " $numpb + 1 " | bc -l `
    460          echo "WARN "$datadiffcounter" files have a different size on data (raw)."
     493      else
     494         result=$result"1"
     495         numpb=`echo " $numpb + 1 " | bc -l `
    461496      fi
    462497      # zipped files
    463498      # dl00
     499      echo "INFO "$dl00okcounter" files are ok on dl00."
     500      echo "WARN "$dl00diffcounter" files have a different size on dl00."
    464501      if [ $dl00okcounter -eq ${dl00[0]} ]
    465502      then
    466503         result=$result"0"
    467504         numok=`echo " $numok + 1 " | bc -l `
    468          echo "INFO "$dl00okcounter" files are ok on dl00."
    469       else
    470          result=$result"1"
    471          numpb=`echo " $numpb + 1 " | bc -l `
    472          echo "WARN "$dl00diffcounter" files have a different size on dl00."
     505      else
     506         result=$result"1"
     507         numpb=`echo " $numpb + 1 " | bc -l `
    473508      fi
    474509      # archive
     510      echo "INFO "$archiveokcounter" files are ok in the archive."
     511      echo "WARN "$archivediffcounter" files have a different size in the archive."
    475512      if [ $archiveokcounter -eq ${daq[0]} ]
    476513      then
    477514         result=$result"0"
    478515         numok=`echo " $numok + 1 " | bc -l `
    479          echo "INFO "$archiveokcounter" files are ok in the archive."
    480       else
    481          result=$result"1"
    482          numpb=`echo " $numpb + 1 " | bc -l `
    483          echo "WARN "$archivediffcounter" files have a different size in the archive."
     516      else
     517         result=$result"1"
     518         numpb=`echo " $numpb + 1 " | bc -l `
    484519      fi
    485520      # wue
     521      echo "INFO "$wueokcounter" files are ok in Wue."
     522      echo "WARN "$wuediffcounter" files have a different size in Wue."
    486523      if [ $wueokcounter -eq ${wue[0]} ]
    487524      then
    488525         result=$result"0"
    489526         numok=`echo " $numok + 1 " | bc -l `
    490          echo "INFO "$wueokcounter" files are ok in Wue."
    491       else
    492          result=$result"1"
    493          numpb=`echo " $numpb + 1 " | bc -l `
    494          echo "WARN "$wuediffcounter" files have a different size in Wue."
    495       fi
    496    fi
     527      else
     528         result=$result"1"
     529         numpb=`echo " $numpb + 1 " | bc -l `
     530      fi
     531   fi
     532   
     533   # list #drs
    497534   
    498535   # print summary:
    … …  
    500537   echo "INFO numok: "$numok
    501538   echo "INFO numpb: "$numpb
    502    echo "result: (#files-dudir-db-filesize)"
    503    echo "ldaw-ldaw-ldaw-ldaw"
    504    echo $result
     539   echo "result:"
     540   echo "(#files-dudir-db-filesize)"
     541   echo " ldaw-ldaw-ldaw-ldaw"
     542   echo " "$result
    505543   #if [ $numpb -eq 0 ]
    506544   if [ $numpb -lt 4 ]
Note: See TracChangeset for help on using the changeset viewer.