source: trunk/DataCheck/Transfer/RsyncRawToISDC.sh @ 18873

Last change on this file since 18873 was 18873, checked in by dorner, 2 years ago
fixed small bug in logging
  • Property svn:executable set to *
File size: 6.2 KB
Line 
1#!/bin/bash
2#
3
4source `dirname $0`/../Sourcefile.sh
5printprocesslog "INFO starting $0"
6program=RsyncRawToISDC
7step=RawFileRsyncedISDC
8rsyncpwfile=/home_nfs/isdc/fact_opr/rsync.pwd
9
10set -C
11
12logfile=$runlogpath"/RsyncRawToISDC-"$datetime".log"
13date >> $logfile 2>&1
14
15
16# check if only a certain date should be processed
17#  (feature for manual speedup)
18if [ "$certaindate" != "" ]
19then
20   printprocesslog "INFO process only data of "$certaindate
21   echo "INFO process only data of "$certaindate
22   specialnight=`echo $certaindate | sed -e 's/\///g'`
23fi
24
25
26# stop transfer when less than 100 GB are left
27function check_scratch()
28{
29   diskusage2=( `df -P /scratch | grep scratch ` )
30   # check if more than X GB are left on /scratch
31   if [ ${diskusage2[3]} -lt 100000 ] 
32   then
33      printprocesslog "DISK less than 100 GB left on /scratch ("${diskusage2[3]}")"
34      echo "WARN less than 100 GB left on /scratch ("${diskusage2[3]}")" 
35      finish
36   fi
37}
38
39check_scratch
40
41# get todo list
42gettodo
43
44for (( s=0 ; s < $num ; s++ ))
45do
46   check_scratch
47   night=${primaries[$s+$s]}
48   runid=${primaries[$s+$s+1]}
49   nightpath=`echo $night | cut -c 1-4`"/"`echo $night | cut -c 5-6`"/"`echo $night | cut -c 7-8`
50   lppath="/data2/zipraw/"$nightpath
51   nightrun=$night"_"`printf %03d $runid`
52
53   # continue if only special night should be processed for manual speedup
54   if [ "$specialnight" != "" ]
55   then
56      if [ $night -ne $specialnight ]
57      then
58         printprocesslog "INFO Treat only "$specialnight" -> continue for "$night
59         echo "INFO Treat only "$specialnight" -> continue for "$night
60         continue
61      else
62         echo "INFO Transfer "$nightrun
63      fi
64   fi
65
66   lpfile=`ssh fact@161.72.93.138 "find $lppath -type f -regex '.*$nightrun[.]fits[.][fg]z'"`
67   if [ "$lpfile" == "" ]
68   then
69      echo $nightrun" not yet available on data."
70      printprocesslog "INFO "$nightrun" not yet available on data."
71      continue
72   else
73      filename=`basename $lpfile`
74   fi
75
76   setstatus "start" 
77
78   rawfilelp="/data2/zipraw/./"$nightpath"/"$filename
79   rawdirisdc="/scratch/from_lapalma/raw/"
80   rawfileisdc=$rawdirisdc$nightpath"/"$filename
81   ingestlinkpath="/scratch/ingest_links/raw/"$nightpath
82   makedir $ingestlinkpath >> $logfile 2>&1
83   ingestlink=$ingestlinkpath"/"$filename
84
85   if ! ssh fact@161.72.93.138 "ls $rawfilelp >/dev/null 2>&1"
86   then
87      #echo $rawfilelp" not yet available on data."
88      printprocesslog "INFO "$rawfilelp" not yet available on data."
89      check="no"
90      setstatus "stop"
91      continue
92   fi
93
94   # for new rsync
95   #   remove / from dir
96   #   change name of dir (else rsync-server doesn't accept it
97   #   test if trick with file/dir is working
98   #echo "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc
99   printprocesslog "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc
100   #echo "INFO rsync rawfile "$rawfilelp" to ISDC "$rawdirisdc >> $logfile 2>&1
101   #rsync -rltDvR --partial --stats --password-file=$rsyncpwfile factdata@161.72.93.138::$rawfilelp $rawdirisdc >> $logfile 2>&1
102   #echo "rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.138:$rawfilelp $rawdirisdc >> $logfile 2>&1"
103   rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.138:$rawfilelp $rawdirisdc >> $logfile 2>&1
104   check1=$?
105
106   case $check1 in
107      0)   printprocesslog "INFO rawfile "$rawfilelp" transferred successfully to ISDC."
108           if ! [ -L $ingestlink ]
109           then 
110              printprocesslog "INFO create link for "$rawfileisdc
111              ln -sv $rawfileisdc $ingestlink >> $logfile 2>&1
112              check2=$?
113              case $check2 in
114                 0)   printprocesslog "INFO link for  "$rawfileisdc" created successfully."
115                      ;;
116                 *)   printprocesslog "WARN link for "$rawfileisdc" could not be created (check2="$check2")."
117                      check=$check2
118                      ;;
119              esac
120           else
121              printprocesslog "INFO link "$ingestlink" already existing."
122           fi
123           ;;
124      *)   printprocesslog "CONNECTION "$rawfilelp" could not be transferred to ISDC (check1="$check1")."
125           check=$check1
126           ;;
127   esac
128   
129   query="SELECT fHasDrsFile from RunInfo WHERE fNight="$night" AND fRunID="$runid
130   numdrs=`sendquery`
131   if [ $numdrs -gt 0 ]
132   then 
133      drsfilelp="/data2/zipraw/./"$nightpath"/"$nightrun".drs.fits.gz"
134      drsfileisdc=$rawdirisdc$nightpath"/"$nightrun".drs.fits.gz"
135      drsingestlink=$ingestlinkpath"/"$nightrun".drs.fits.gz"
136
137      if ! ssh fact@161.72.93.138 "ls $drsfilelp >/dev/null 2>&1"
138      then
139         #echo $drsfilelp" not yet available on data."
140         printprocesslog "INFO "$drsfilelp" not yet available on data."
141         check="no"
142         setstatus "stop"
143         continue
144      fi
145
146
147      printprocesslog "INFO rsync drsfile "$drsfilelp" to ISDC "$rawdirisdc
148      echo "INFO rsync drsfile "$drsfilelp" to ISDC "$rawdirisdc >> $logfile 2>&1
149      #rsync -rltDvR --partial --stats --password-file=$rsyncpwfile factdata@161.72.93.138::$rawfilelp $rawdirisdc >> $logfile 2>&1
150      rsync -rltDvR -T $rsynctempdir --partial --stats fact@161.72.93.138:$drsfilelp $rawdirisdc >> $logfile 2>&1
151      check3=$?
152
153      case $check3 in
154         0)   printprocesslog "INFO drsfile "$drsfilelp" transferred successfully to ISDC."
155              if ! [ -L $drsingestlink ]
156              then 
157                 printprocesslog "INFO create link for "$drsfileisdc
158                 ln -sv $drsfileisdc $drsingestlink >> $logfile 2>&1
159                 check4=$?
160                 case $check4 in
161                    0)   printprocesslog "INFO link for  "$drsfileisdc" created successfully."
162                         ;;
163                    *)   printprocesslog "WARN link for "$drsfileisdc" could not be created (check4="$check4")."
164                         check=$check4
165                         ;;
166                 esac
167              else
168                 printprocesslog "INFO link "$drsingestlink" already existing."
169              fi
170              ;;
171         *)   printprocesslog "CONNECTION "$drsfilelp" could not be transferred to ISDC (check3="$check3")."
172              check=$check3
173              ;;
174      esac
175   fi
176
177   setstatus "stop" 
178done
179
180finish
181
Note: See TracBrowser for help on using the repository browser.