#!/bin/sh
#
# ========================================================================
#
# *
# * This file is part of MARS, the MAGIC Analysis and Reconstruction
# * Software. It is distributed to you in the hope that it can be a useful
# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
# * It is distributed WITHOUT ANY WARRANTY.
# *
# * Permission to use, copy, modify and distribute this software and its
# * documentation for any purpose is hereby granted without fee,
# * provided that the above copyright notice appear in all copies and
# * that both that copyright notice and this permission notice appear
# * in supporting documentation. It is provided "as is" without express
# * or implied warranty.
# *
#
#
#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
#
#   Copyright: MAGIC Software Development, 2000-2004
#
#
# ========================================================================
#
#
# This script handles the subsystem logfiles, which are copied every
# day automatically from pc15 in La Palma to /data/MAGIC/transfer/ in
# Wuerzburg
# The structure in this directory is: subsystem/date/logfile.*
# The logfiles are from CC (*.rep, CC*.dbg, CC*.rbk, CC*.rep, CC*.run,
# CC*.run.html) and Cosy (cosy*.log, cosy*.rep, tpoint*.txt/starg*.txt) 
# AMC (AMC*.log, AMC*.info, AMC*.lut) and CaCo (dc*.txt) are not copied
# every day automatically
#
# the script performs the following steps:
# ----------------------------------------
# - filling the information from the *.rbk and *.run files into the DB
# using the macros filldotrbk.C and filldotrun.C
# - copying all logfiles to the correct directory:
# /Period*/subsystem/date/file.*
# (the period for each date is obtained by the macro getperiod.C) 
# - filling the sequences for the standard analysis into the database
# (for each day of which a file was copied) 
# 
# 
# -- to make sure that no file is copied twice there's a textfile
# (.processed), to which every file, that was processed, is added 
# .processed is copied to La Palma, so that it is possible to copy
# only new files
# -- to make sure that a file was copied completely before it is
# processed, a file with the checksum of all files is also copied from
# La Palma and the checksum is checked with this script for each file
# before processing it. 
# 
#

export ROOTSYS=/opt/root_v3.10.02
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ROOTSYS/lib
export PATH=$PATH:$ROOTSYS/bin

set -C

mars=/home/operator/Mars
macrospath=$mars/datacenter/macros
scriptspath=$mars/datacenter/scripts
transdir=/magic/datacenter/transfer
char=28
extern=MAGIC@virgo:/data/MAGIC/transfer
datetime=`date +%F-%H-%M-%S`

lockpath=/magic/datacenter/locks
logpath=/magic/datacenter/autologs

scriptlogpath=$logpath/run/copyscript/`date +%Y/%m/%d`
if [ ! -d $scriptlogpath ]
then
  mkdir -pv $scriptlogpath 
  if [ ! -d $scriptlogpath ]
  then 
     echo "could not make scriptlogpath "$scriptlogpath 
     exit
  fi
fi

scriptlog=$scriptlogpath/copyscript-$datetime.log

date >> $scriptlog 2>&1

date > $lockpath/lock-copyscript.txt >> $scriptlog 2>&1 
checklock0=$?
case $checklock0 in 
    0)   echo "checklock0=$checklock0 -> continue " >> $scriptlog 2>&1;;
    1)   echo "checklock0=$checklock0 -> file exists " >> $scriptlog 2>&1
         echo "-> copyscript is running -> exit" >> $scriptlog 2>&1
         date  >> $scriptlog 2>&1
         exit;;
    *)   echo "checklock0=$checklock0 -> something went completely wrong" >> $scriptlog 2>&1;;
esac

# rsync new dc with virgo
rsync -av $extern/ $transdir  >> $scriptlog 2>&1 

cd $transdir

#check the checksums, write output into file
md5sum -c .checksums | tee $transdir/.check  >> $scriptlog 2>&1 
#md5sum -c $transdir/checksums-test >| $transdir/.check 
if ! test -e $transdir/.check
then
   echo "couldn't create checkfile -> exit script" >> $scriptlog 2>&1
   rm -v $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
   date  >> $scriptlog 2>&1
   exit
else
   echo "checkfile created" >> $scriptlog 2>&1
fi

cd $mars 

echo "remove empty directories: "  >> $scriptlog 2>&1 
rmdir -pv $transdir/*/*  >> $scriptlog 2>&1 

#find all dates in transfer-directory
#dates=`ls $transdir/*/ -1 | grep --regexp=20[0-9][0-9]_*[0-1][0-9]_*[0-3][0-9] | grep -v '.tgz'`
dates=`ls $transdir/*/ -1 | grep --regexp=20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] | grep -v '.tgz'` >> $scriptlog 2>&1

if [ "$dates" = "" ]
then 
   echo "nothing to do -> exit"  >> $scriptlog 2>&1
   rm -v $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
   date  >> $scriptlog 2>&1
   exit
fi

echo "dates found: "$dates  >> $scriptlog 2>&1 

#loop over dates 
for date in ${dates[@]}
do 
   #find all files for this date
   files=`find $transdir/*/$date/ -name "*.*"`  >> $scriptlog 2>&1
   if [ "$files" = "" ]
   then 
      echo "no files found for $date"  >> $scriptlog 2>&1
      continue
   else
      echo "files found for $date"  >> $scriptlog 2>&1
   fi
   
   date2=`echo $date | sed -e 's/_/\//g'`
#   echo "date2=$date2"
   copylogpath=$logpath/copyscript/$date2
   echo "copylogpath: "$copylogpath >> $scriptlog 2>&1
   if [ ! -d $copylogpath ]
   then
     mkdir -pv $copylogpath >> $scriptlog 2>&1
     if [ ! -d $copylogpath ]
     then
        echo "could not make copylogpath $copylogpath -> continue " >> $scriptlog 2>&1
        continue
     fi
   fi
   
   #loop over all files in the directories for this date
   for file in $files
   do
      echo "file: "$file  >> $scriptlog 2>&1 
      filename=`basename $file`
#      echo "filename: "$filename
                        
      #check for each file whether it has to be processed
      if grep "$filename: OK" $transdir/.check  >> $scriptlog 2>&1 
      then
         newfile=`echo $file | sed -e 's/datacenter\/transfer/subsystemdata/' -e 's/ccdata/cc/' -e 's/cacodata/caco/' -e 's/drivelogs/drive/' -e 's/_/\//1' -e 's/_/\//1' `
         echo "file (new path): "$newfile >> $scriptlog 2>&1 
         newpath=`dirname $newfile`
#         echo "new path: "$newpath
         #make sure, that the needed directories are available
         if [ ! -d $newpath ]
         then
            mkdir -pv $newpath >> $scriptlog 2>&1 
            if [ ! -d $newpath ]
            then 
               echo "could not make path $newpath"  >> $scriptlog 2>&1 
               continue
            fi
         fi
         
         #run macros if the file is a *.rbk or a *.run file to
         #fill the information into the db
         if echo $filename | grep .rbk  >> $scriptlog 2>&1 
         then
            echo "found rbk-file $filename"  >> $scriptlog 2>&1 
            checkfillrbk=`root -q -b $macrospath/filldotrbk.C+\("\"$file\""\,kFALSE\) | tee $copylogpath/filldotrbk-$filename-log.txt | grep int | sed -e 's/(int)//'`
            case $checkfillrbk in
               1) echo "checkfillrbk=$checkfillrbk -> everything ok " >> $scriptlog 2>&1 
                  echo " -> go on with copying file" >> $scriptlog 2>&1 ;;
               *) echo "checkfillrbk=$checkfillrbk - Error -> go on with next file" >> $scriptlog 2>&1 
                  continue;;
            esac
         fi
         if echo $file | grep 'CC_.*.run' >> $scriptlog 2>&1  && ! echo $file | grep .run.html >> $scriptlog 2>&1 
         then
            echo "found run-file $filename" >> $scriptlog 2>&1 
            checkfillrun=`root -q -b $macrospath/filldotrun.C+\("\"$file\""\,kFALSE\) | tee $copylogpath/filldotrun-$filename-log.txt  | grep int | sed -e 's/(int)//'`
            case $checkfillrun in
               1) echo "checkfillrun=$checkfillrun -> everything ok " >> $scriptlog 2>&1 
                  echo "-> insert date in SequenceBuildStatus for $date" >> $scriptlog 2>&1 
#                  date3=`echo $date | sed -e 's/_/-/g'`
#                  echo "date3=$date3"
                  checkinsertdate=`root -q -b $macrospath/insertdate.C+\("\"$date\""\) | tee $copylogpath/insertdate-$date-log.txt   | grep int | sed -e 's/(int)//'`
                  case $checkinsertdate in
                     1) echo "date inserted" >> $scriptlog 2>&1 ;;
                     *) echo "checkinsertdate=$checkinsertdate -> ERROR - insert date didn't work" >> $scriptlog 2>&1 
                        continue;;
                  esac
                  echo " -> go on with copying file" >> $scriptlog 2>&1 ;;
               *) echo "checkfillrun=$checkfillrun Error -> go on with next file" >> $scriptlog 2>&1 
                  continue;;
            esac
         fi

         #copy file
         if mv -v $file $newpath >> $scriptlog 2>&1 
         then
            #add file to donelist
            filename3=`echo $file | cut -c $char-999`
            echo "$filename3 processed" >> $transdir/.processed
            echo $filename" moved and added to .processed" >> $scriptlog 2>&1 
         else
            echo $file" couldn't be moved!!!!!" >> $scriptlog 2>&1 
         fi

      else
         #if the checksum for the file is not ok or the file was
         #already processed (entry in donelist) -> go on with next file
         echo $file": checksum not ok " >> $scriptlog 2>&1 
      fi
   done
done

echo "remove empty directories: "  >> $scriptlog 2>&1 
rmdir -pv $transdir/*/*  >> $scriptlog 2>&1 

rm -v $transdir/.check >> $scriptlog 2>&1 

#rsync -av $transdir/ $extern >> $scriptlog 2>&1 
rsync -av --delete $transdir/ $extern >> $scriptlog 2>&1 

rm -v $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
date >> $scriptlog 2>&1

set +C
