source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9620

Last change on this file since 9620 was 9618, checked in by Daniela Dorner, 14 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 15.7 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58 if [ ! -d $@ ]
59 then
60 mkdir -pv $@
61 if [ ! -d $@ ]
62 then
63 if ! [ "$processlog" = "" ]
64 then
65 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66 else
67 echo "could not make dir "$@
68 fi
69 if ls $lockfile >/dev/null 2>&1
70 then
71 rm -v $lockfile
72 fi
73 exit
74 fi
75 fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91 makedir $processlogpath
92 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99 then
100 printprocesslog "INFO " `rm -v $lockfile`
101 fi
102 printprocesslog "INFO finished $0"
103 exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110# setup for jobmanager:
111# log files (can't be defined in script itself, as script can run longer
112# than one day
113jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
114jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
115
116# check if rc-files are available
117if ! ls $steps >/dev/null
118then
119 echo "Can't find steps.rc ($steps)"
120 finish
121fi
122if ! ls $sqlrc >/dev/null
123then
124 echo "Can't find sql.rc ($sqlrc)"
125 finish
126fi
127
128# resetting values for jobmanager
129pno=0
130totalpno=0
131running=0
132queued=0
133runningscript=0
134queuedscript=0
135stillinqueue=0
136
137
138# alias (we cannot check the beginning of the line due to
139# color codes in filldotraw.C)
140alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
141
142
143# in the following the functions, which are needed by several scripts, are
144# defined
145
146# function to check if a process is already locked
147# command line option can be used to execute something, e.g. 'continue'
148function checklock()
149{
150 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
151 then
152 printprocesslog "WARN lockfile $lockfile exists"
153 $@
154 exit
155 else
156 printprocesslog "INFO created lockfile $lockfile"
157 fi
158}
159
160# print the current status values
161function printstatusvalues()
162{
163 echo "the current values are:"
164 echo " starttime=$starttime"
165 echo " stoptime=$stoptime"
166 echo " returncode=$returncode"
167 echo "-- check: -$check-"
168 echo ""
169}
170
171# get the db-setup from the sql.rc
172function getdbsetup()
173{
174 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
175 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
176 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
177 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
178# echo "setup: "
179# echo " db: "$db
180# echo " pw: "$pw
181# echo " us: "$us
182# echo " ho: "$ho
183}
184
185# function to get the needed information from the dependencies-file steps.rc
186function getstepinfo()
187{
188 getdbsetup
189 needs=`grep "$step[.]Needs:" $steps | grep -v '#' | sed -e "s/$step[.]Needs://"`
190 noderestricted=`grep "$step[.]NodeRestricted:" $steps | grep -v '#' | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
191 prims=( `grep "$step[.]Primaries:" $steps | grep -v '#' | sed -e "s/$step[.]Primaries://"` )
192# echo " needs: $needs"
193# echo " noderestricted: $noderestricted"
194# echo " prims: ${prims[@]}"
195}
196
197# function to get the primaries of a step from the dependencies-file steps.rc
198function getprimary()
199{
200 getdbsetup
201 grep $@"[.]Primaries:" $steps | grep -v '#' | sed -e "s/$@[.]Primaries://"
202}
203
204# function to get the join of a step
205# from the dependencies-file steps.rc
206# $1 gives the step
207# $2 gives the type of join
208function getjoin()
209{
210 getdbsetup
211 grep $1"[.]"$2"Join:" $steps | grep -v '#' | sed -e "s/$1[.]$2Join://"
212}
213
214# function to get the needs of a step from the dependencies-file steps.rc
215function getneeds()
216{
217 getdbsetup
218 grep $@"[.]Needs:" $steps | sed -e "s/$@[.]Needs://"
219}
220
221# subquery which is needed to check the
222# status of the steps with different primaries
223function subquerypart()
224{
225 query=$query" SELECT COUNT(*) FROM "$need"Status "`getjoin $need "SpecialInfluence"`
226 needinfluences=`grep $need $steps | grep "Needs" | grep -v "$need[.]Needs" | cut -d'.' -f1`
227 for needinfl in $needinfluences
228 do
229 needinfljoin=`getjoin $needinfl "Influence"`
230 if ! [ "$needinfljoin" = "" ]
231 then
232 query=$query" LEFT JOIN "$needinfl"Status USING ("$needinfljoin") "
233 fi
234 query=$query" "`getjoin $needinfl "SpecialInfluence"`" "
235 done
236 query=$query" WHERE "`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
237 for (( j=1 ; j < ${#prims[@]} ; j++ ))
238 do
239 query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
240 done
241}
242
243# function to create the middle part of a query
244# which is identical for the functions getstatus() and gettodo()
245function middlepartofquery()
246{
247 # add from which table the information is queried
248 query=$query" FROM "$step"Status "
249 # add the joins to the tables in which the status of the preceding steps is stored
250 for need in $needs
251 do
252 needprims=( `getprimary $need` )
253 # make sure that the correct joins are used
254 specialneedjoin=`getjoin $need "SpecialNeed"`
255 if [ "$specialneedjoin" = "" ]
256 then
257 query=$query" LEFT JOIN "$need"Status USING (${needprims[@]}) "
258 else
259 query=$query" "$specialneedjoin" "
260 query=$query" LEFT JOIN "$need"Status USING (${needprims[@]}) "
261 break
262 fi
263 done
264 # add condition
265 query=$query" WHERE "
266 # add condition for the status of the peceding steps
267 counter=0
268 specialneedjoin=
269 specialinfljoin=
270 for need in $needs
271 do
272 if [ $counter -gt 0 ]
273 then
274 query=$query" AND "
275 fi
276 specialneedjoin=$specialneedjoin" "`getjoin $need "SpecialNeed"`
277 specialinfljoin=$specialinfljoin" "`getjoin $need "SpecialInfluence"`
278 if [ "$specialneedjoin" = " " ]
279 then
280 # condition for step with same primaries
281 query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
282 query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
283 query=$query" ISNULL("$need"Status.fReturnCode) "
284 specialneedjoin=
285 specialinfljoin=
286 else
287 # special query to check steps which
288 # do not have the same primaries
289 # comparison of
290 # number of rows
291 # with
292 # number of rows which are done
293 query=$query" ( "
294 subquerypart
295 query=$query" ) = ( "
296 subquerypart
297 query=$query" AND NOT ISNULL("$need"Status.fStartTime) "
298 query=$query" AND NOT ISNULL("$need"Status.fStopTime) "
299 query=$query" AND ISNULL("$need"Status.fReturnCode)) "
300 fi
301 counter=`echo $counter + 1 | bc -l`
302 done
303 # add condition for the status of the step itself
304 if [ $counter -gt 0 ]
305 then
306 query=$query" AND "
307 fi
308 query=$query" ISNULL("$step"Status.fStartTime) "
309 query=$query" AND ISNULL("$step"Status.fStopTime) "
310 query=$query" AND ISNULL("$step"Status.fReturnCode) "
311}
312
313# function to get todolist
314# returns the next or the list of next steps
315function gettodo()
316{
317 # reset the variable for the number of the next step
318 process=
319 printprocesslog "INFO getting todo..."
320 getstepinfo
321 # get query
322 query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
323 for (( j=1 ; j < ${#prims[@]} ; j++ ))
324 do
325 query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
326 done
327 # get middle par of query
328 middlepartofquery
329 # add requirement for production host in case it is needed
330 if [ "$2 " != " " ]
331 then
332 query=$query" AND fProductionHostKEY=$2 "
333 fi
334 # order by priority to the the number of the next step to be done
335 query=$query" ORDER BY "$step"Status.fPriority desc "
336 # add limitation in case only one or a limited number of
337 # processes should be executed
338 if [ "$1 " != " " ]
339 then
340 query=$query" limit 0, $1 "
341 fi
342 # print query
343 #echo " gettodo QUERY: "$query
344 printprocesslog "INFO gettodo QUERY: "$query
345 # execute query
346 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
347 then
348 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
349 finish
350 fi
351 # get numbers of next step from mysql result
352 if [ "$process" = "" ]
353 then
354 printprocesslog "INFO => nothing to do"
355 finish
356 else
357 primaries=( $process )
358 num=`expr ${#primaries[@]} / ${#prims[@]} `
359 fi
360}
361
362# function to get the number of processes which still have to be done
363function getstatus()
364{
365 # reset the variable for the number of steps to be done
366 numproc=
367 getstepinfo
368 # get query
369 query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
370 # get middle part of query
371 middlepartofquery
372 # add requirement for production host in case it is needed
373 if [ "$1 " != " " ]
374 then
375 query=$query" AND fProductionHostKEY=$1 "
376 fi
377 # group by an 'artifical' column to get the number of lines
378 query=$query" GROUP BY 2 "
379 # printing query
380 #echo " getstatus QUERY: "$query
381 printprocesslog "INFO getstatus QUERY: "$query
382 # execute query
383 if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
384 then
385 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
386 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
387 continue
388 fi
389 # get number of processes from mysql result
390 if [ "${numprocs[1]}" = "" ]
391 then
392 numproc=0
393 else
394 numproc=${numprocs[0]}
395 fi
396}
397
398# function to set status of a process in the db
399function setstatus()
400{
401 # remark:
402 # this function does not include the 'Default' flag
403 # for resetting steps
404
405 # for dowebplots (there are steps which have no entry in the DB)
406 if [ "$step" = "no" ]
407 then
408 return
409 fi
410
411 # reset status values
412 starttime=NULL
413 stoptime=NULL
414 returncode=NULL
415 # evaluate the status values
416 case $@ in
417 start) printprocesslog "INFO setstatus start"
418 starttime="Now()"
419 ;;
420 stop) case $check in
421 ok) printprocesslog "INFO setstatus stop - ok"
422 starttime=noreset
423 stoptime="Now()"
424 ;;
425 no) printprocesslog "INFO setstatus stop - nothing new"
426 check="ok"
427 ;;
428 *) printprocesslog "INFO setstatus stop - failed"
429 starttime=noreset
430 stoptime="Now()"
431 if [ "$check" == "" ]
432 then
433 returncode=1
434 else
435 returncode=$check
436 fi
437 check="ok"
438 ;;
439 esac
440 ;;
441 *) printprocesslog "ERROR function setstatus got wrong variable"
442 finish
443 ;;
444 esac
445 # get
446 getstepinfo
447
448 # get the influences from the steps.rc by evaluating the needs of all steps
449 influences=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
450
451 # get query
452 query=" UPDATE "$step"Status "
453 # add joins to the influenced tables
454 specialinfljoin=`getjoin $step "SpecialInfluence"`
455 if ! [ "$specialinfljoin" = "" ]
456 then
457 query=$query" "$specialinfljoin
458 fi
459 for influence in $influences
460 do
461 infljoin=`getjoin $influence "Influence"`
462 if [ "$infljoin" = "" ]
463 then
464 inflprims=( `getprimary $influence` )
465 query=$query" LEFT JOIN "$influence"Status USING (${inflprims[@]}) "
466 else
467 query=$query" LEFT JOIN "$influence"Status USING ("$infljoin") "
468 fi
469 specialinfljoin2=`getjoin $influence "SpecialInfluence"`
470 if ! [ "$specialinfljoin2" = "" ]
471 then
472 query=$query" "$specialinfljoin2
473 fi
474 done
475 # set the status values according to the new status of the step
476 query=$query" SET "
477 if ! [ "$starttime" = "noreset" ]
478 then
479 query=$query" "$step"Status.fStartTime=$starttime, "
480 fi
481 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
482 # set also the status values of the influenced steps
483 for influence in $influences
484 do
485 query=$query", "$influence"Status.fStartTime=NULL "
486 query=$query", "$influence"Status.fStopTime=NULL "
487 query=$query", "$influence"Status.fReturnCode=NULL "
488 done
489 # give the condition for which step the status values have to be set
490 query=$query" WHERE "
491 if [ "$s" = "" ]
492 then
493 s=0
494 fi
495 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
496 for (( j=1 ; j < ${#prims[@]} ; j++ ))
497 do
498 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
499 done
500 # print query
501 #echo " setstatus QUERY: "$query
502 printprocesslog "INFO setstatus QUERY: "$query
503 # execute query
504 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
505 then
506 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
507 finish
508 fi
509}
510
511# function to send a mysql query
512function sendquery()
513{
514 getdbsetup
515 printprocesslog "INFO sendquery QUERY: "$query
516 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
517 then
518 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
519 return 1
520 fi
521 if [ "$val" = "NULL" ]
522 then
523 val=
524 fi
525 echo $val
526 return 0
527}
528
Note: See TracBrowser for help on using the repository browser.