source: trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles@ 8235

#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2006
#
#
# ========================================================================
#
# This script is moving the rawfiles from the directory
# /magic/datacenter/fromtape/rawdata to the structure
# /magic/data/rawfiles/YYYY/MM/DD
# 

source `dirname $0`/sourcefile
printprocesslog "INFO starting $0"
program=movingrawfiles

set -C
umask 0002

scriptlog=$runlogpath/$program-$datetime.log
date >> $scriptlog 2>&1

# check whether script is already running
lockfile=$lockpath/lock-$program.txt
checklock  >> $scriptlog 2>&1

# change permission for files
ssh tape@dc07 chmod -R g+w /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1
# output for chmod with -v or -c (only changes)

printprocesslog "INFO moving directories from fromlapalma to fromtape"
#find directories which have to be copied
dirs=`ls /magic/datacenter/fromlapalma/RAWchk/*/*.finished | sed -e 's/RAWchk/RAW/g' | cut -d/ -f1-6`
if [ ! "$dirs" == "" ]
then
   for dir in $dirs
   do
      # get number of *.raw files from directory
      numfiles=`find $dir -type f -name "*.raw" | wc -l`
      
      # skip empty directories
      if [ "$numfiles" == "0" ]
      then
         continue
      fi

      # get date from directory name
      curdate=`echo $dir | cut -d/ -f6- | sed -e 's/_/-/g'`
      
      getdbsetup
      # get number of runs on this date from db
      query="select COUNT(*) from RunData where ADDDATE(fRunStart, Interval 12 hour) like '$curdate %'"

      if ! numruns=`mysql -s -u $us --password=$pw --host=hercules $db -e "$query" | tail -1`
      then
         echo "ERROR could not query number of runs from rundb -> continue" >> $scriptlog 2>&1 
         printprocesslog "ERROR could not query number of runs for $curdate from rundb"
         continue
      fi

      # Check if number of runs matches. If not, skip this directory
      if ! [ $numfiles -eq $numruns ]
      then
         echo "WARNING" >> $scriptlog 2>&1 
         echo "Expecting $numruns raw-files for $curdate but found only $numfiles" >> $scriptlog 2>&1 
         continue
      fi

      # check for non-raw-files
      nonraw=`find $dir -type f ! -name "*.raw"`
      if [ ! "$nonraw" == "" ]
      then
         echo "non-raw-files found in $dir" >> $scriptlog 2>&1 
         printprocesslog "WARNING non-raw-files found in $dir"
      fi

      # change permission for this directory
      ssh lapalma@apollo chmod -R g+w $dir >> $scriptlog 2>&1
      # move directories to the tapedirectory
      mv -v $dir /magic/datacenter/fromtape/rawdata/ >> $scriptlog 2>&1 

   done
fi


# find rawfiles
rawfiles=`find /magic/datacenter/fromtape/rawdata/ -name '*.*'`

if [ "$rawfiles" == "" ]
then
   echo "no files to move -> exit" >> $scriptlog 2>&1 
   finish >> $scriptlog 2>&1 
fi

printprocesslog "INFO moving moving rawfiles to $datapath/rawfiles"
for rawfile in $rawfiles
do 
# workaround for rawfiles with wrong timing
#  newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles-wrong-timing/g' -e 's/_/\//1' -e 's/_/\//1'`
   newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'`
   newdir=`dirname $newrawfile`
   makedir $newdir >> $scriptlog 2>&1 

   mv -v $rawfile $newrawfile >> $scriptlog 2>&1 
done

rmdir -v /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1 

printprocesslog "INFO launching filesondisk"
echo "launching filesondisk" >> $scriptlog 2>&1 
$scriptspath/filesondisk&

finish >> $scriptlog 2>&1