Index: trunk/DataCheck/QuickLook/Step1.sh
===================================================================
--- trunk/DataCheck/QuickLook/Step1.sh	(revision 18806)
+++ trunk/DataCheck/QuickLook/Step1.sh	(revision 18855)
@@ -3,5 +3,4 @@
 
 # remarks: 
-# rsync-server still used
 # move fileerror check to main-loop?
 
@@ -20,12 +19,11 @@
    datepath=$certaindate
 else
-   datepath=`date --date="-19HOUR" +%Y/%m/%d`
+   # this has to be coordinated with the time when 
+   #   ClearNewdata.sh is running
+   datepath=`date --date="-18HOUR" +%Y/%m/%d`
 fi
 date=`echo $datepath | sed -e 's/\///g'`
 printprocesslog "INFO processing "$datepath
 
-auxpathnewdaq=/newdaq/aux/$datepath
-#auxpathnewdaq=/loc_data/aux/$datepath #to run on newdaq
-# create aux directory on daq, if not yet there
 auxpath=/loc_data/aux/$datepath
 makedir $auxpath >/dev/null
@@ -38,5 +36,4 @@
 makedir $calpath >/dev/null
 rawpathnewdaq=/newdaq/raw/$datepath
-#rawpathnewdaq=/loc_data/raw/$datepath #to run on newdaq
 rawpath=/loc_data/raw/$datepath
 
@@ -44,20 +41,14 @@
 #   drive file with information about current source position
 drivefile=$auxpath/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
-drivefilenewdaq=$auxpathnewdaq/${date}.DRIVE_CONTROL_SOURCE_POSITION.fits
 #   drive file with information tracking position
 drivefile2=$auxpath/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
-drivefilenewdaq2=$auxpathnewdaq/${date}.DRIVE_CONTROL_TRACKING_POSITION.fits
 #   file with magic weather information 
 mweatherfile=$auxpath/${date}.MAGIC_WEATHER_DATA.fits
-mweatherfilenewdaq=$auxpathnewdaq/${date}.MAGIC_WEATHER_DATA.fits
 #   file with trigger rates
 ratesfile=$auxpath/${date}.FTM_CONTROL_TRIGGER_RATES.fits
-ratesfilenewdaq=$auxpathnewdaq/${date}.FTM_CONTROL_TRIGGER_RATES.fits
 #   file with trigger rates
 tempfile=$auxpath/${date}.FSC_CONTROL_TEMPERATURE.fits
-tempfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_TEMPERATURE.fits
 #   file with trigger rates
 humfile=$auxpath/${date}.FSC_CONTROL_HUMIDITY.fits
-humfilenewdaq=$auxpathnewdaq/${date}.FSC_CONTROL_HUMIDITY.fits
 
 function rsync_aux_file()
@@ -66,12 +57,5 @@
    then 
       printprocesslog "INFO rsync "$1
-      # rsync 
-      #   from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/
-      #   to daq (/daq = /loc_data on daq)
-      rsyncservernewdaq=`echo $1 | sed -e 's/^\//172.16.100.100::/'` 
-      # old
-      if ! rsync -a -T $rsynctempdir $1 $2
-      # new (workaround for problems on daq)
-      #if ! rsync -a -T $rsynctempdir $rsyncservernewdaq $2
+      if ! rsync -a -T $rsynctempdir newdaq:$1 $1
       then 
          printprocesslog "WARN rsync of "$1" failed."
@@ -80,24 +64,33 @@
 }
 
-function check_daq()
+function check_disks()
 {
-   diskusage=( `df -P /raid10 | grep raid10 ` )
-   # check if more than 700 GB are left on /loc_data
-   if [ ${diskusage[3]} -lt $disklimitdaq ] 
+   # put here the check for /scratch (output of rsync) and /data1 (output of analysis)
+   
+   # at least 10% free disk on /scratch
+   diskusage=( `df -P /scratch | grep scratch ` )
+   if [ ${diskusage[3]} -lt $disklimitnewdata ] 
    then
-      echo "WARN less than 700 left on /raid10 on node "$HOSTNAME
-      printprocesslog "WARN less than 700 left on /raid10 on node "$HOSTNAME
-      df -h /raid10
+      echo "WARN less than 10% left on /scratch on node "$HOSTNAME
+      printprocesslog "WARN less than 10% left on /scratch on node "$HOSTNAME
+      df -h /scratch
       finish
    fi
+   # at least 5% free disk on /data1
+   diskusage=( `df -P /data1 | grep data1 ` )
+   if [ ${diskusage[3]} -lt $disklimitnewdata2 ] 
+   then
+      echo "WARN less than 5% left on /data1 on node "$HOSTNAME
+      printprocesslog "WARN less than 5% left on /data1 on node "$HOSTNAME
+      df -h /scratch
+      finish
+   fi
 }
 
-check_daq
-
+check_disks
+
+# getting lists of files
 printprocesslog "INFO get lists of raw files on newdaq and daq"
-files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
-# to treat links use:
-#files=( `find -L $rawpathnewdaq -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
-
+files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
 if [ ${#files[@]} -eq 0 ]
 then
@@ -105,7 +98,7 @@
    finish
 fi
-fileslocal=( `find -L $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null| sort` )
-
-callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null| sort` )
+fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
+callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
+
 # get number of dataruns from DB
 query="SELECT Count(*) FROM RunInfo WHERE fNight="$date" AND fRunTypeKey=1"
@@ -124,6 +117,6 @@
 makedir $rawpath >/dev/null
 
-#echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
-printprocesslog "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
+#echo "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
+printprocesslog "INFO #files-local:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
 
 while [ ${#fileslocal[@]} -ne ${#files[@]} ] || [ $numcalibrated -ne ${#callistofiles[@]} ] # || [ $numcalibrated -ne 0 ] # FIXME: Logik ueberdenken u ueberarb
@@ -132,5 +125,5 @@
    #  when there is more than 10% space on daq
    source `dirname $0`/../Sourcefile.sh
-   check_daq
+   check_disks
    
    numcalibrated=0
@@ -138,6 +131,6 @@
    printprocesslog "INFO status beginning of while-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
 
-   rsync_aux_file $drivefilenewdaq $drivefile
-   rsync_aux_file $drivefilenewdaq2 $drivefile2
+   rsync_aux_file $drivefile
+   rsync_aux_file $drivefile2
    
    # files on newdaq
@@ -145,6 +138,7 @@
    do
       printprocesslog "processing "$file
-      #echo "processing "$file
-      localfile=`echo $file | sed -e 's/newdaq/loc_data/'`
+      echo "processing "$file
+      #localfile=`echo $file | sed -e 's/newdaq/scratch/'`
+      localfile=$file
 
       source `dirname $0`/../Sourcefile.sh
@@ -158,5 +152,5 @@
          then
             nondrs=`basename $file | sed -e 's/[.]drs//g'`
-            nondrsfile=`find -L $rawpath -name $nondrs.*z`
+            nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null `
             tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
          else
@@ -170,5 +164,5 @@
             # if a file is not closed and not touched for 30 minutes, 
             #    it is assumed corrupted and still transferred
-            fileaccessed=`find $file -amin -30`
+            fileaccessed=`find $file -amin -30 2>/dev/null `
             if ! [ "$fileaccessed" == "" ]
             then
@@ -182,14 +176,5 @@
 
          # rsync 
-         #   from newdaq (/newdaq = /fact on newdaq), rsync server newdaq::newdaq/
-         #   to daq (/daq = /loc_data on daq)
-         # to access rsync server via the dedicated network between 
-         #    daq and newdaq, use 172.16.100.100::newdaq
-         filersyncserver=`echo $file | sed -e 's/^\//172.16.100.100::/'` 
-         # old
-         ##if ! rsync -av --stats --progress --bwlimit=$bwlimit $file $localfile
-         if ! rsync -a -T $rsynctempdir --bwlimit=$bwlimit $file $localfile
-         # new
-         #if ! rsync -a -W -T $rsynctempdir --bwlimit=$bwlimit $filersyncserver $localfile
+         if ! rsync -au -T $rsynctempdir --bwlimit=$bwlimit newdaq:$file $localfile
          then
             printprocesslog "ERROR something went wrong with rsync of "$file
@@ -210,5 +195,5 @@
 #      then
 #         nondrs=`basename $file | sed -e 's/[.]drs//g'`
-#         nondrsfile=`find -L $rawpath -name $nondrs.*z`
+#         nondrsfile=`find -L $rawpath -name $nondrs.*z 2>/dev/null `
 #         tstop=`$factpath/fitsdump -h $nondrsfile  2>/dev/null | grep TSTOPI | grep -E -o '[0-9]+'`
 #      else
@@ -282,10 +267,10 @@
             if ! [ -e $callistolog ]
             then
-               rsync_aux_file $drivefilenewdaq $drivefile
-               rsync_aux_file $drivefilenewdaq2 $drivefile2
-               rsync_aux_file $mweatherfilenewdaq $mweatherfile
-               rsync_aux_file $ratesfilenewdaq $ratesfile
-               rsync_aux_file $tempfilenewdaq $tempfile
-               rsync_aux_file $humfilenewdaq $humfile
+               rsync_aux_file $drivefile
+               rsync_aux_file $drivefile2
+               rsync_aux_file $mweatherfile
+               rsync_aux_file $ratesfile
+               rsync_aux_file $tempfile
+               rsync_aux_file  $humfile
                
                calfile=$calpath"/"$date"_"$runnum"_C.root"
@@ -366,7 +351,7 @@
    # get new file lists
    printprocesslog "INFO get new file lists for "$datepath
-   files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
-   fileslocal=( `find -L $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
-   callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` )
+   files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
+   fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
+   callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
    #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
    printprocesslog "INFO status after for-loop #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
@@ -379,7 +364,7 @@
       sleep 60
       printprocesslog "INFO get new file lists for "$datepath
-      files=( `find $rawpathnewdaq -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
-      fileslocal=( `find -L $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' | sort` )
-      callistofiles=( `find $calpath -type f -name $date*-calibration.log | sort` )
+      files=( `ssh newdaq "find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort "` )
+      fileslocal=( `find $rawpath -type f -regex '.*20[0-9][0-9][01][0-9][0-3][0-9][_][0-9][0-9][0-9][.]d?r?s?[.]?fits[.]?[g]?[f]?[z]?' 2>/dev/null | sort` )
+      callistofiles=( `find $calpath -type f -name $date*-calibration.log 2>/dev/null | sort` )
    fi
    #echo "INFO #files-daq:"${#fileslocal[@]}" #files-newdaq:"${#files[@]}" #callisto-logs:"${#callistofiles[@]}" #runs:"$numcalibrated
@@ -390,3 +375,3 @@
 
 
-
+finish