source: trunk/DataCheck/Archive/rawIngest.sh@ 20012

#!/bin/bash

sourceFolder=$1 
destFolder=$2 

if [ "$#" != "3" ]
then
        echo "Please specify source and dest folders. and an identifier for the log files Aborting"
        exit
fi

if [ $1 == "" ]
then
        echo "Source folder is empty. Aborting"
        exit
fi

if [ $2 == "" ]
then
        echo "Dest folder is empty. Aborting"
        exit
fi

if [ $3 == "" ]
then 
        echo "Identifier for log files empty. Aborting"
        exit
fi

#first let's make sure that source and dest folders do exist, and that dest is writable
if [ -d $1 ]
then
        sourceFolder=$1
else
        echo "Source folder "$1" does not exist (or cannnot be read.) Aborting"
        exit
fi

if [ -d $2 ]
then
        if [ -d $2"/etiennetest" ]
        then
                echo "Test folder already exist. Aborting"
                exit
        fi
        mkdir $2"/etiennetest" 2>/dev/null
        if [ -d $2"/etiennetest" ]
        then
                rm -rf $2"/etiennetest"
                destFolder=$2
        else
                echo "Dest folder is not writable. Aborting"
                exit
        fi
else
        echo "Dest folder does not exist. Aborting"
        exit
fi

#files=`ls $destFolder`
#if [ "$files" != "" ]
#then
#       echo "Dest folder is not empty. Aborting"
#       exit
#fi
sourceFolder=${sourceFolder%/}
destFolder=${destFolder%/}
echo "Will start ingesting files from "$sourceFolder" to "$destFolder
echo "Will start ingesting files from "$sourceFolder" to "$destFolder >> Rawreport$3.txt

#list all the files in sourceFolder, and copy then with the same structure to destfolder

entries=`find $sourceFolder -type f -name '*.fits' | sort`

for entry in ${entries[@]}
do
       #second construct the destination path.
        filenameonly=${entry##*/}
        #first construct the correct file name
        targetFileName=$filenameonly
        pathonly=${entry%$filenameonly}
        extrapathonly=${pathonly#$sourceFolder/}
        targetFolder=$destFolder"/"$extrapathonly
        if [ ! -d $targetFolder ]
        then
                mkdir -p $targetFolder
        fi
        
        #check if the file already exist there
        targetFile=$targetFolder"/"$targetFileName
        interFile="/scratch/"$targetFileName
        echo "$targetFile"
#       echo "$interFile"
        if [ -a $targetFile".gz" ]
        then
                echo "File $targetFile already exist. Skipping it" >> Rawreport$3.txt
                continue
        fi
        cp $entry $interFile
        #if not, do the copying, fixing and checking

#       grouping=`/home/isdc/lyard/FACT++/fitsdump $targetFile -h 2>/dev/null | grep GROUPING`
        
#       grouping=`grep 'GROUPING' "temp.txt"`
                
#       if [ "$grouping" == "" ]
#       then
                
        repairRawFile.sh $interFile RawENDerrors$3.txt RawMJDerror$3.txt Rawreport$3.txt RawprocessErrors$3.txt
        
        if [ -a $interFile ]
        then
                fixRawKeyWords.sh $interFile RawreportTwo$3.txt RawprocessErrors$3.txt
                result=`fverify $interFile 2>/dev/null | grep '0 error(s)'`
                if [ "$result" == "" ]
                then
                        echo "$interFile" >> RawstillHasProblems$3.txt
                        rm $interFile
                else
                        gzip -1 $interFile
                        cp $interFile".gz" $targetFile".gz"
                        rm $interFile".gz"
                fi
        fi
done

#set the correct permissions
find $destFolder -type f -exec chmod 640 {} \;
find $destFolder -type d -exec chmod 750 {} \;
find $destFolder -exec chgrp fact {} \;