source: branches/Mars_MC/datacenter/scripts/sourcefile@ 17038

Last change on this file since 17038 was 10041, checked in by Daniela Dorner, 14 years ago
lock in db instead of lockfile
  • Property svn:executable set to *
File size: 13.7 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46if [ "$SOURCEFILEPATH" = "" ]
47then
48 export SOURCEFILEPATH=`dirname $0`
49fi
50if [ "$SCRIPTNAME" = "" ]
51then
52 SCRIPTNAME=`basename $0`
53fi
54
55source $SOURCEFILEPATH/setup.$AUTOMATIONSETUP
56
57datetime=`date +%F-%H-%M-%S`
58
59
60# function to make sure that a directory is made
61function makedir()
62{
63 if [ ! -d $@ ]
64 then
65 mkdir -pv $@
66 if [ ! -d $@ ]
67 then
68 if ! [ "$processlog" = "" ]
69 then
70 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
71 else
72 echo "could not make dir "$@
73 fi
74 if ls $lockfile >/dev/null 2>&1
75 then
76 rm -v $lockfile
77 fi
78 exit
79 fi
80 fi
81}
82
83# logging paths for runlogs and processlog
84runlogpath=$logpath/run/`date +%Y/%m/%d`
85processlogpath=$logpath/processlog
86makedir $runlogpath
87makedir $processlogpath
88processlog=$processlogpath/process`date +%F`.log
89
90makedir $lockpath
91
92
93# function to provide proper logging in a single logfile ($processlog)
94function printprocesslog
95{
96 makedir $processlogpath
97 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
98}
99
100# function to exit a script properly
101function finish()
102{
103 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
104 then
105 printprocesslog "DEBUG " `rm -v $lockfile`
106 fi
107 printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
108 exit
109}
110
111
112# set checkvalue to ok at the beginning of the scripts
113check="ok"
114
115# setup for jobmanager:
116# log files (can't be defined in script itself, as script can run longer
117# than one day
118jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
119jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
120
121# check if rc-files are available
122if ! ls $steps >/dev/null
123then
124 echo "Can't find steps.rc ($steps)"
125 finish
126fi
127if ! ls $sqlrc >/dev/null
128then
129 echo "Can't find sql.rc ($sqlrc)"
130 finish
131fi
132
133# resetting values for jobmanager
134pno=0
135totalpno=0
136running=0
137queued=0
138runningscript=0
139queuedscript=0
140stillinqueue=0
141
142
143# alias (we cannot check the beginning of the line due to
144# color codes in filldotraw.C)
145alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
146
147
148# in the following the functions, which are needed by several scripts, are
149# defined
150
151# function to check if a process is already locked
152# command line option can be used to execute something, e.g. 'continue'
153function checklock()
154{
155 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
156 then
157 printprocesslog "WARN lockfile $lockfile exists"
158 $@
159 exit
160 else
161 printprocesslog "DEBUG created lockfile $lockfile"
162 fi
163}
164
165# print the current status values
166function printstatusvalues()
167{
168 echo "the current values are:"
169 echo " starttime=$starttime"
170 echo " stoptime=$stoptime"
171 echo " availtime=$availtime"
172 echo " returncode=$returncode"
173 echo "-- check: -$check-"
174 echo ""
175}
176
177# get the db-setup from the sql.rc
178function getdbsetup()
179{
180 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
181 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
182 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
183 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
184# echo "setup: "
185# echo " db: "$db
186# echo " pw: "$pw
187# echo " us: "$us
188# echo " ho: "$ho
189}
190
191# function to get information from the setupfile $steps
192function getfromsetup()
193{
194 grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
195}
196
197# function to get the needed information from the dependencies-file steps.rc
198function getstepinfo()
199{
200 getdbsetup
201 needs=( `getfromsetup $step "Needs"` )
202 noderestricted=`getfromsetup $step "NodeRestricted"`
203 prims=( `getfromsetup $step "Primaries"` )
204 maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
205# echo " maintable: "$maintable
206# echo " needs: "${needs[@]}
207# echo " noderestricted: "$noderestricted
208# echo " prims: "${prims[@]}
209}
210
211# function to get the joins needed for the get/set status queries
212function getalljoins()
213{
214 # add table
215 query=$query" "$maintable"Status"
216 # add special join
217 query=$query" "`getfromsetup $maintable "SpecialJoin"`
218 # add join for step unless step is the same as maintable
219 if ! [ "$step" = "$maintable" ]
220 then
221 query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
222 fi
223 # add joins for influences or needs
224 for otherstep in ${othersteps[@]}
225 do
226 if ! [ "$otherstep" = "$maintable" ]
227 then
228 query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
229 fi
230 done
231}
232
233# function to create the middle part of a query
234# which is identical for the functions getstatus() and gettodo()
235function getstatusquery()
236{
237 # add from which table the information is queried
238 query=$query" FROM "
239 othersteps=${needs[@]}
240 getalljoins
241 # add condition
242 query=$query" WHERE "
243 # add condition for step, i.e. step is not yet done
244 query=$query" ISNULL("$step"Status.fStartTime) "
245 query=$query" AND ISNULL("$step"Status.fStopTime) "
246 query=$query" AND ISNULL("$step"Status.fAvailable) "
247 query=$query" AND ISNULL("$step"Status.fReturnCode) "
248 # add requirement for production host in case it is needed
249 if [ "$1 " != " " ]
250 then
251 query=$query" AND fProductionHostKEY=$2 "
252 fi
253 if ! echo $query | grep UPDATE >/dev/null 2>&1
254 then
255 query=$query" GROUP BY "${prims[@]}
256 fi
257 # add condition for needs, i.e. that step is done
258 for (( k=0 ; k < ${#needs[@]} ; k++ ))
259 do
260 if [ $k -eq 0 ]
261 then
262 query=$query" HAVING "
263 else
264 query=$query" AND "
265 fi
266 query=$query" COUNT(*)=COUNT(IF("
267 query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
268 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
269 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
270 query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
271 query=$query" , 1, NULL)) "
272 done
273}
274
275# function to get todolist
276# returns the next or the list of next steps
277function gettodo()
278{
279 # reset the variable for the number of the next step
280 process=
281 printprocesslog "DEBUG getting todo for step $step..."
282 getstepinfo
283 # get query
284 query=" SELECT "${prims[@]}
285 getstatusquery $2
286 # order by priority to the the number of the next step to be done
287 query=$query" ORDER BY "$step"Status.fPriority desc "
288 # add limitation in case only one or a limited number of
289 # processes should be executed
290 if [ "$1 " != " " ]
291 then
292 query=$query" limit 0, $1 "
293 fi
294 # print query
295 printprocesslog "DEBUG gettodo for step $step QUERY: "$query
296 # execute query
297 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
298 then
299 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
300 finish
301 fi
302 # get numbers of next step from mysql result
303 if [ "$process" = "" ]
304 then
305 printprocesslog "DEBUG => nothing to do"
306 finish
307 else
308 primaries=( $process )
309 num=`expr ${#primaries[@]} / ${#prims[@]} `
310 fi
311}
312
313# function to get the number of processes which still have to be done
314function getstatus()
315{
316 # reset the variable for the number of steps to be done
317 numproc=0
318 getstepinfo
319 # get query
320 query=" SELECT "${prims[@]}
321 getstatusquery $1
322 # print query
323 printprocesslog "DEBUG getstatus for step $step QUERY: "$query
324 # execute query
325 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query " | wc -l`
326 then
327 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
328 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
329 continue
330 fi
331}
332
333# function to set status of a process in the db
334function setstatus()
335{
336 # remark:
337 # this function does not include the 'Default' flag
338 # for resetting steps
339
340 # for dowebplots (there are steps which have no entry in the DB)
341 if [ "$step" = "no" ]
342 then
343 return
344 fi
345
346 # reset status values
347 starttime=NULL
348 stoptime=NULL
349 availtime=NULL
350 returncode=NULL
351 # evaluate the status values
352 case $@ in
353 start) printprocesslog "DEBUG setstatus start"
354 starttime="Now()"
355 ;;
356 stop) case $check in
357 ok) printprocesslog "DEBUB setstatus stop - ok"
358 starttime=noreset
359 stoptime="Now()"
360 if [ "$processingsite" = "$storagesite" ]
361 then
362 availtime="Now()"
363 fi
364 ;;
365 no) printprocesslog "DEBUG setstatus stop - nothing new"
366 check="ok"
367 ;;
368 *) printprocesslog "DEBUG setstatus stop - failed"
369 starttime=noreset
370 stoptime="Now()"
371 if [ "$processingsite" = "$storagesite" ]
372 then
373 availtime="Now()"
374 fi
375 if [ "$check" == "" ]
376 then
377 returncode=1
378 else
379 returncode=$check
380 fi
381 check="ok"
382 ;;
383 esac
384 ;;
385 *) printprocesslog "ERROR function setstatus got wrong variable"
386 finish
387 ;;
388 esac
389
390 # get
391 getstepinfo
392
393 # get the influences from the steps.rc by evaluating the needs of all steps
394 othersteps=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
395
396 # get query
397 query=" UPDATE "
398 getalljoins
399 # set the status values according to the new status of the step
400 query=$query" SET "
401 if ! [ "$starttime" = "noreset" ]
402 then
403 query=$query" "$step"Status.fStartTime=$starttime, "
404 fi
405 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
406 query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
407 # set also the status values of the influenced steps
408 for otherstep in $othersteps
409 do
410 query=$query", "$otherstep"Status.fStartTime=NULL "
411 query=$query", "$otherstep"Status.fStopTime=NULL "
412 query=$query", "$otherstep"Status.fAvailable=NULL "
413 query=$query", "$otherstep"Status.fReturnCode=NULL "
414 query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
415 done
416 # give the condition for which step the status values have to be set
417 query=$query" WHERE "
418 if [ "$s" = "" ]
419 then
420 s=0
421 fi
422 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
423 for (( j=1 ; j < ${#prims[@]} ; j++ ))
424 do
425 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
426 done
427 # add additional query to allow for locking in db
428 if [ "$1" = "start" ]
429 then
430 query=$query" AND ISNULL("$step"Status.fStartTime) "
431 fi
432 # add row count to know how many rows have been changed
433 query=$query"; SELECT ROW_COUNT();"
434 # print query
435 printprocesslog "DEBUG setstatus for step $step QUERY: "$query
436 # execute query
437 if ! numchanged=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
438 then
439 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
440 finish
441 fi
442 if [ $numchanged -gt 0 ]
443 then
444 printprocesslog "INFO successful setting of status in db (program: $program, function setstatus)"
445 else
446 printprocesslog "INFO status in db was already set by other process (program: $program, function setstatus)"
447 fi
448}
449
450# function to send a mysql query
451function sendquery()
452{
453 getdbsetup
454 printprocesslog "DEBUG sendquery QUERY: "$query
455 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
456 then
457 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
458 return 1
459 fi
460 if [ "$val" = "NULL" ]
461 then
462 val=
463 fi
464 echo $val
465 return 0
466}
467
Note: See TracBrowser for help on using the repository browser.