Index: trunk/DataCheck/Transfer/CheckTransfer.sh =================================================================== --- trunk/DataCheck/Transfer/CheckTransfer.sh (revision 17078) +++ trunk/DataCheck/Transfer/CheckTransfer.sh (revision 17079) @@ -85,9 +85,9 @@ dates=( `ssh fact@161.72.93.131 "find /loc_data/zipraw -mindepth 3 -type d | sort | sed -e 's/\/loc_data\/zipraw\///g' "` ) -dates=( "2013/05/23" ) +#dates=( "2013/08/02" ) #short="yes" numdaysok=0 -numdaysoklimit=5 +numdaysoklimit=10 sumdata=0 checklimit=5000 @@ -130,4 +130,5 @@ localfailpath="/gpfs/fact/fact-archive/fails/raw/"$date wuerawpath="/fact/raw/"$date + qlapath="/daq/analysis/callisto/"$date #phidorawpath="/fhgfs/groups/app/fact-construction/raw/"$date @@ -147,4 +148,5 @@ wue=( `ssh operator@coma.astro.uni-wuerzburg.de "if [ -d $wuerawpath ]; then ls $wuerawpath/* | wc -l; du -s -b --apparent-size $wuerawpath; else echo '-1 -1 -1'; fi"` ) #phido=( `ssh -i /home_nfs/isdc/fact_opr/.ssh/id_rsa.fact_opr.phido 129.217.160.201 "if [ -d $phidorawpath ]; then ls $phidorawpath/* | wc -l; du -s -b --apparent-size $phidorawpath; else echo '-1 -1 -1'; fi"` ) + qla=( `ssh fact@161.72.93.131 "ls $qlapath/20*_C.root 2>/dev/null | wc -l"` ) query="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM RunInfo WHERE fNight="$date2 querystart="SELECT Sum(if(fHasDrsFile=1,2,1)) FROM " @@ -180,4 +182,9 @@ # numphido=0 #fi + + # select number of data runs + query="SELECT COUNT(*) FROM RunInfo WHERE fNight="$date2 + query=$query" AND fRunTypeKey=1" + numdatruns=`sendquery` printprocesslog "disk: " @@ -190,4 +197,5 @@ printprocesslog " fail "${fails[@]} #printprocesslog " phido "${phido[@]} + printprocesslog " qla "${qla[@]} printprocesslog "db: " printprocesslog " runinfo "$numruns @@ -196,4 +204,5 @@ printprocesslog " wue "$numwue #printprocesslog " phido "$numphido + printprocesslog " data "$numdatruns echo "disk: " >> $logfile 2>&1 echo " newdaq "${newdaq[@]} >> $logfile 2>&1 @@ -205,4 +214,5 @@ echo " fail "${fails[@]} >> $logfile 2>&1 #echo " phido "${phido[@]} >> $logfile 2>&1 + echo " qla "${qla[@]} >> $logfile 2>&1 echo "db: " >> $logfile 2>&1 echo " runinfo "$numruns >> $logfile 2>&1 @@ -211,5 +221,14 @@ echo " wue "$numwue >> $logfile 2>&1 #echo " phido "$numphido >> $logfile 2>&1 - + echo " datruns "$numdatruns >> $logfile 2>&1 + + if ! [ $numdatruns -eq $qla ] + then + printprocesslog "WARN not all data runs are processed yet by the QLA for "$date"." + echo "WARN not all data runs are processed yet by the QLA for "$date >> $logfile 2>&1 + result1="1-" + else + result1="0-" + fi # check if file are available in the different places if [ ${dl00[0]} -eq -1 ] && [ $date2 -lt 20120308 ] @@ -231,8 +250,8 @@ echo "WARN number of files on daq (" ${daq[0]}") does not agree with number of files on newdaq (" ${newdaq[0]}") for "$date >> $logfile 2>&1 numpb=`echo " $numpb + 1 " | bc -l ` - result1="1" + result1=$result1"1" else numok=`echo " $numok + 1 " | bc -l ` - result1="0" + result1=$result1"0" fi # dl00 @@ -304,5 +323,5 @@ #fi - if [ "$short" = "yes" ] || [ "$result1" != "0000" ] + if [ "$short" = "yes" ] || [ "$result1" != "0-0000" ] then printprocesslog "number of files does not yet agree in all sites ("$result1") -> do no further checking." @@ -756,7 +775,7 @@ printprocesslog "INFO numpb: "$numpb printprocesslog "result:" - printprocesslog "(#files-dudir-db-filesize)" + printprocesslog "(qla-#files-dudir-db-filesize)" #printprocesslog " ldawp-ldawp-ldawp-ldawp" - printprocesslog " ldaw-ldaw-ldaw-ldaw" + printprocesslog " q-ldaw-ldaw-ldaw-ldaw" printprocesslog " "$result1"-"$result2"-"$result3"-"$result4 echo "INFO day ok: "$numdaysok >> $logfile 2>&1 @@ -764,7 +783,7 @@ echo "INFO numpb: "$numpb >> $logfile 2>&1 echo "result:" >> $logfile 2>&1 - echo "(#files-dudir-db-filesize)" >> $logfile 2>&1 + echo "(qla-#files-dudir-db-filesize)" >> $logfile 2>&1 #echo " ldawp-ldawp-ldawp-ldawp" >> $logfile 2>&1 - echo " ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1 + echo " q-ldaw-ldaw-ldaw-ldaw" >> $logfile 2>&1 echo " "$result1"-"$result2"-"$result3"-"$result4 >> $logfile 2>&1 sumdatanew=`echo " ( ${daq[1]} + ${zip[1]} ) / 1024 / 1024 / 1024 " | bc -l | cut -d. -f1` @@ -776,4 +795,5 @@ fi + echo "" echo "SUMMARY for "$date echo "-----------------------" @@ -789,5 +809,5 @@ #echo "arch:"${archive[0]} #echo "isdc:"$numisdc - if [ "$result1" = "0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ] + if [ "$result1" = "0-0000" ] && [ "$result3" = "0000" ] && [ "$result4" = "0000" ] then echo " EVERYTHING is ok. "$date" can be deleted. " @@ -797,6 +817,7 @@ echo " Details in the logfile "$logfile >> $logfile 2>&1 echo " Details in the logfile "$logfile >> $logfile2 2>&1 - else - if [ "$result1" = "0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ] + sendemail="yes" + else + if [ "$result1" = "0-0000" ] && [ "$result3" = "0010" ] && [ "$result4" = "0000" ] && [ ${archive[0]} -eq $numisdc ] then echo " "${fails[0]}" file(s) corrupt (fails folder), but files are transfered correctly. " @@ -809,4 +830,5 @@ echo " Details in the logfile "$logfile >> $logfile 2>&1 echo " Details in the logfile "$logfile >> $logfile2 2>&1 + sendemail="yes" else echo " "$date" is not yet transfered completely. Please check the logfile "$logfile @@ -841,4 +863,10 @@ echo " WARN "$archivediffcounter" files have a different size in the archive." >> $logfile 2>&1 echo " WARN "$archivediffcounter" files have a different size in the archive." >> $logfile2 2>&1 + fi + if ! [ $numdatruns -eq $qla ] + then + echo "WARN not all data runs are processed yet by the QLA. " + echo "WARN not all data runs are processed yet by the QLA. " >> $logfile 2>&1 + echo "WARN not all data runs are processed yet by the QLA. " >> $logfile2 2>&1 fi done