source: trunk/MagicSoft/Mars/datacenter/scripts/copyscript@ 6935

Last change on this file since 6935 was 6935, checked in by Daniela Dorner, 20 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 9.1 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2004
23#
24#
25# ========================================================================
26#
27#
28# This script handles the subsystem logfiles, which are copied every
29# day automatically from pc15 in La Palma to /data/MAGIC/transfer/ in
30# Wuerzburg
31# The structure in this directory is: subsystem/date/logfile.*
32# The logfiles are from CC (*.rep, CC*.dbg, CC*.rbk, CC*.rep, CC*.run,
33# CC*.run.html) and Cosy (cosy*.log, cosy*.rep, tpoint*.txt/starg*.txt)
34# AMC (AMC*.log, AMC*.info, AMC*.lut) and CaCo (dc*.txt) are not copied
35# every day automatically
36#
37# the script performs the following steps:
38# ----------------------------------------
39# - filling the information from the *.rbk and *.run files into the DB
40# using the macros filldotrbk.C and filldotrun.C
41# - copying all logfiles to the correct directory:
42# /Period*/subsystem/date/file.*
43# (the period for each date is obtained by the macro getperiod.C)
44# - filling the sequences for the standard analysis into the database
45# (for each day of which a file was copied)
46#
47#
48# -- to make sure that no file is copied twice there's a textfile
49# (.processed), to which every file, that was processed, is added
50# .processed is copied to La Palma, so that it is possible to copy
51# only new files
52# -- to make sure that a file was copied completely before it is
53# processed, a file with the checksum of all files is also copied from
54# La Palma and the checksum is checked with this script for each file
55# before processing it.
56#
57#
58
59export ROOTSYS=/opt/root_v3.10.02
60export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ROOTSYS/lib
61export PATH=$PATH:$ROOTSYS/bin
62
63set -C
64
65mars=/home/operator/Mars
66macrospath=$mars/datacenter/macros
67scriptspath=$mars/datacenter/scripts
68transdir=/magic/datacenter/transfer
69extern=MAGIC@virgo:/data/MAGIC/transfer
70datetime=`date +%F-%H-%M-%S`
71
72lockpath=/magic/datacenter/locks
73logpath=/magic/datacenter/autologs
74
75scriptlogpath=$logpath/run/copyscript/`date +%Y/%m/%d`
76if [ ! -d $scriptlogpath ]
77then
78 mkdir -pv $scriptlogpath
79 if [ ! -d $scriptlogpath ]
80 then
81 echo "could not make scriptlogpath "$scriptlogpath
82 exit
83 fi
84fi
85
86scriptlog=$scriptlogpath/copyscript-$datetime.log
87
88date >> $scriptlog 2>&1
89
90date > $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
91checklock0=$?
92case $checklock0 in
93 0) echo "checklock0=$checklock0 -> continue " >> $scriptlog 2>&1;;
94 1) echo "checklock0=$checklock0 -> file exists " >> $scriptlog 2>&1
95 echo "-> copyscript is running -> exit" >> $scriptlog 2>&1
96 date >> $scriptlog 2>&1
97 exit;;
98 *) echo "checklock0=$checklock0 -> something went completely wrong" >> $scriptlog 2>&1;;
99esac
100
101# rsync new dc with virgo
102rsync -av $extern/ $transdir >> $scriptlog 2>&1
103
104cd $transdir
105
106#check the checksums, write output into file
107md5sum -c .checksums | tee $transdir/.check >> $scriptlog 2>&1
108#md5sum -c $transdir/checksums-test >| $transdir/.check
109if ! test -e $transdir/.check
110then
111 echo "couldn't create checkfile -> exit script" >> $scriptlog 2>&1
112 rm -v $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
113 date >> $scriptlog 2>&1
114 exit
115else
116 echo "checkfile created" >> $scriptlog 2>&1
117fi
118
119cd $mars
120
121echo "remove empty directories: " >> $scriptlog 2>&1
122rmdir -pv $transdir/*/* >> $scriptlog 2>&1
123
124#find all dates in transfer-directory
125#dates=`ls $transdir/*/ -1 | grep --regexp=20[0-9][0-9]_*[0-1][0-9]_*[0-3][0-9] | grep -v '.tgz'`
126dates=`ls $transdir/*/ -1 | grep --regexp=20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] | grep -v '.tgz'` >> $scriptlog 2>&1
127
128if [ "$dates" = "" ]
129then
130 echo "nothing to do -> exit" >> $scriptlog 2>&1
131 rm -v $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
132 date >> $scriptlog 2>&1
133 exit
134fi
135
136echo "dates found: "$dates >> $scriptlog 2>&1
137
138#loop over dates
139for date in ${dates[@]}
140do
141 #find all files for this date
142 files=`find $transdir/*/$date/ -name "*.*"` >> $scriptlog 2>&1
143 if [ "$files" = "" ]
144 then
145 echo "no files found for $date" >> $scriptlog 2>&1
146 continue
147 else
148 echo "files found for $date" >> $scriptlog 2>&1
149 fi
150
151 date2=`echo $date | sed -e 's/_/\//g'`
152# echo "date2=$date2"
153 copylogpath=$logpath/copyscript/$date2
154 echo "copylogpath: "$copylogpath >> $scriptlog 2>&1
155 if [ ! -d $copylogpath ]
156 then
157 mkdir -pv $copylogpath >> $scriptlog 2>&1
158 if [ ! -d $copylogpath ]
159 then
160 echo "could not make copylogpath $copylogpath -> continue " >> $scriptlog 2>&1
161 continue
162 fi
163 fi
164
165 #loop over all files in the directories for this date
166 for file in $files
167 do
168 echo "file: "$file >> $scriptlog 2>&1
169 filename=`basename $file`
170# echo "filename: "$filename
171
172 #check for each file whether it has to be processed
173 if grep "$filename: OK" $transdir/.check >> $scriptlog 2>&1
174 then
175 newfile=`echo $file | sed -e 's/datacenter\/transfer/subsystemdata/' -e 's/ccdata/cc/' -e 's/drivelogs/drive/' -e 's/_/\//1' -e 's/_/\//1' `
176 echo "file (new path): "$newfile >> $scriptlog 2>&1
177 newpath=`dirname $newfile`
178# echo "new path: "$newpath
179 #make sure, that the needed directories are available
180 if [ ! -d $newpath ]
181 then
182 mkdir -pv $newpath >> $scriptlog 2>&1
183 if [ ! -d $newpath ]
184 then
185 echo "could not make path $newpath" >> $scriptlog 2>&1
186 continue
187 fi
188 fi
189
190 #run macros if the file is a *.rbk or a *.run file to
191 #fill the information into the db
192 if echo $filename | grep .rbk >> $scriptlog 2>&1
193 then
194 echo "found rbk-file $filename" >> $scriptlog 2>&1
195 checkfillrbk=`root -q -b $macrospath/filldotrbk.C+\("\"$file\""\,kFALSE\) | tee $copylogpath/filldotrbk-$date-log.txt | grep int | sed -e 's/(int)//'`
196 case $checkfillrbk in
197 1) echo "checkfillrbk=$checkfillrbk -> everything ok " >> $scriptlog 2>&1
198 echo " -> go on with copying file" >> $scriptlog 2>&1 ;;
199 *) echo "checkfillrbk=$checkfillrbk - Error -> go on with next file" >> $scriptlog 2>&1
200 continue;;
201 esac
202 fi
203 if echo $file | grep 'CC_.*.run' >> $scriptlog 2>&1 && ! echo $file | grep .run.html >> $scriptlog 2>&1
204 then
205 echo "found run-file $filename" >> $scriptlog 2>&1
206 checkfillrun=`root -q -b $macrospath/filldotrun.C+\("\"$file\""\,kFALSE\) | tee $copylogpath/filldotrun-$date-log.txt | grep int | sed -e 's/(int)//'`
207 case $checkfillrun in
208 1) echo "checkfillrun=$checkfillrun -> everything ok " >> $scriptlog 2>&1
209 echo "-> insert date in SequenceBuildStatus for $date" >> $scriptlog 2>&1
210# date3=`echo $date | sed -e 's/_/-/g'`
211# echo "date3=$date3"
212 checkinsertdate=`root -q -b $macrospath/insertdate.C+\("\"$date\""\) | tee $copylogpath/insertdate-$date-log.txt | grep int | sed -e 's/(int)//'`
213 case $checkinsertdate in
214 1) echo "date inserted" >> $scriptlog 2>&1 ;;
215 *) echo "checkinsertdate=$checkinsertdate -> ERROR - insert date didn't work" >> $scriptlog 2>&1
216 continue;;
217 esac
218 echo " -> go on with copying file" >> $scriptlog 2>&1 ;;
219 *) echo "checkfillrun=$checkfillrun Error -> go on with next file" >> $scriptlog 2>&1
220 continue;;
221 esac
222 fi
223
224 #copy file
225 if mv -v $file $newpath >> $scriptlog 2>&1
226 then
227 #add file to donelist
228 echo "$file processed" >> $transdir/.processed
229 echo $filename" moved and added to .processed" >> $scriptlog 2>&1
230 else
231 echo $file" couldn't be moved!!!!!" >> $scriptlog 2>&1
232 fi
233
234 else
235 #if the checksum for the file is not ok or the file was
236 #already processed (entry in donelist) -> go on with next file
237 echo $file": checksum not ok " >> $scriptlog 2>&1
238 fi
239 done
240done
241
242echo "remove empty directories: " >> $scriptlog 2>&1
243rmdir -pv $transdir/*/* >> $scriptlog 2>&1
244
245rm -v $transdir/.check >> $scriptlog 2>&1
246
247rsync -av --delete $transdir/ $extern >> $scriptlog 2>&1
248
249rm -v $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
250date >> $scriptlog 2>&1
251
252set +C
Note: See TracBrowser for help on using the repository browser.