source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9605

Last change on this file since 9605 was 9605, checked in by Daniela Dorner, 15 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 15.7 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58 if [ ! -d $@ ]
59 then
60 mkdir -pv $@
61 if [ ! -d $@ ]
62 then
63 if ! [ "$processlog" = "" ]
64 then
65 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66 else
67 echo "could not make dir "$@
68 fi
69 if ls $lockfile >/dev/null 2>&1
70 then
71 rm -v $lockfile
72 fi
73 exit
74 fi
75 fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91 makedir $processlogpath
92 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99 then
100 printprocesslog "INFO " `rm -v $lockfile`
101 fi
102 printprocesslog "INFO finished $0"
103 exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110#failed codes
111#sequence build status
112Fbuildsequ=1
113Fdoexcl=2
114#run process status
115Ftimecorr=3
116Ffillraw=4
117Fsinope=5
118Ffillsinope=6
119Fresetexcl=7
120#sequence process status
121Fwritesequfile=8
122Ffilesavail=9
123Fnoccfile=10
124Fnocacofile=11
125Fmerppcc=12
126Fmerppcaco=13
127Fcallisto=14
128Ffillcalib=15
129Ffillsignal=16
130Fstar=17
131Ffillstar=18
132#dataset process status
133Fwritedatasetfile=19
134Fstardone=20
135Fganymed=21
136Ffillganymed=22
137#again run process status
138FCompmux=26
139Fdowebplots=27
140#again mc run process status
141Fmccallisto=28
142Ffillmccalib=29
143Ffillmcsignal=30
144Fmcstar=31
145Ffillmcstar=32
146Fcorsikasimtel=33
147Fchimp=34
148Fchimpcp=35
149Fctastar=36
150Fctastarcp=37
151FctastereoA=38
152FctastereoB=39
153FctastereoC=40
154FctastereoD=41
155FctastereoE=42
156FctastereoF=43
157FctastereoG=44
158FctastereoH=45
159Fctastereocp=46
160
161# setup for jobmanager:
162# log files (can't be defined in script itself, as script can run longer
163# than one day
164jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
165jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
166
167# check if rc-files are available
168if ! ls $steps >/dev/null
169then
170 echo "Can't find steps.rc ($steps)"
171 finish
172fi
173if ! ls $sqlrc >/dev/null
174then
175 echo "Can't find sql.rc ($sqlrc)"
176 finish
177fi
178
179# resetting values
180pno=0
181totalpno=0
182running=0
183queued=0
184runningscript=0
185queuedscript=0
186stillinqueue=0
187
188
189# alias (we cannot check the beginning of the line due to
190# color codes in filldotraw.C)
191alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
192
193
194# in the following the functions, which are needed by several scripts, are
195# defined
196
197# function to check if a process is already locked
198# command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202 then
203 printprocesslog "WARN lockfile $lockfile exists"
204 $@
205 exit
206 else
207 printprocesslog "INFO created lockfile $lockfile"
208 fi
209}
210
211# print the current status values
212function printstatusvalues()
213{
214 echo "the current values are:"
215 echo " starttime=$starttime"
216 echo " stoptime=$stoptime"
217 echo " returncode=$returncode"
218 echo "-- check: -$check-"
219 echo ""
220}
221
222# get the db-setup from the sql.rc
223function getdbsetup()
224{
225 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
226 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
227 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
228 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
229# echo "setup: "
230# echo " db: "$db
231# echo " pw: "$pw
232# echo " us: "$us
233# echo " ho: "$ho
234}
235
236# function to get the needed information from the dependencies-file steps.rc
237function getstepinfo()
238{
239 getdbsetup
240 needs=`grep "$step[.]Needs:" $steps | grep -v '#' | sed -e "s/$step[.]Needs://"`
241 noderestricted=`grep "$step[.]NodeRestricted:" $steps | grep -v '#' | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
242 prims=( `grep "$step[.]Primaries:" $steps | grep -v '#' | sed -e "s/$step[.]Primaries://"` )
243# echo " needs: $needs"
244# echo " noderestricted: $noderestricted"
245# echo " prims: ${prims[@]}"
246}
247
248# function to get the primaries of a step from the dependencies-file steps.rc
249function getprimary()
250{
251 getdbsetup
252 grep $@"[.]Primaries:" $steps | grep -v '#' | sed -e "s/$@[.]Primaries://"
253}
254
255# function to get the join of a step
256# from the dependencies-file steps.rc
257# $1 gives the step
258# $2 gives the type of join
259function getjoin()
260{
261 getdbsetup
262 grep $1"[.]"$2"Join:" $steps | grep -v '#' | sed -e "s/$1[.]$2Join://"
263}
264
265# function to get the needs of a step from the dependencies-file steps.rc
266function getneeds()
267{
268 getdbsetup
269 grep $@"[.]Needs:" $steps | sed -e "s/$@[.]Needs://"
270}
271
272# subquery which is needed to check the
273# status of the steps with different primaries
274function subquerypart()
275{
276 query=$query" SELECT COUNT(*) FROM "$need"Status "
277 query=$query" "`getjoin $need "SpecialInfluence"`" "
278 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
279 for (( j=1 ; j < ${#prims[@]} ; j++ ))
280 do
281 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
282 done
283}
284
285# function to create the middle part of a query
286# which is identical for the functions getstatus() and gettodo()
287function middlepartofquery()
288{
289 # add from which table the information is queried
290 query=$query" FROM "$step"Status "
291 # add the joins to the tables in which the status of the preceding steps is stored
292 for need in $needs
293 do
294 needprims=( `getprimary $need` )
295 # make sure that the correct joins are used
296 specialneedjoin=`getjoin $need "SpecialNeed"`
297 if [ "$specialneedjoin" = "" ]
298 then
299 query=$query" LEFT JOIN "$need"Status USING (${needprims[@]}) "
300 else
301 query=$query" "$specialneedjoin" "
302 query=$query" LEFT JOIN "$need"Status USING (${needprims[@]}) "
303 break
304 fi
305 done
306 # add condition
307 query=$query" WHERE "
308 # add condition for the status of the peceding steps
309 counter=0
310 specialneedjoin=
311 for need in $needs
312 do
313 if [ $counter -gt 0 ]
314 then
315 query=$query" AND "
316 fi
317 specialneedjoin=$specialneedjoin" "`getjoin $need "SpecialNeed"`
318 if [ "$specialneedjoin" = " " ]
319 then
320 # condition for step with same primaries
321 query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
322 query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
323 query=$query" ISNULL("$need"Status.fReturnCode) "
324 specialneedjoin=
325 else
326 # special query to check steps which
327 # do not have the same primaries
328 # comparison of
329 # number of rows
330 # with
331 # number of rows which are done
332 query=$query" ( "
333 subquerypart
334 query=$query" ) = ( "
335 subquerypart
336 query=$query" AND NOT ISNULL(fStartTime) "
337 query=$query" AND NOT ISNULL(fStopTime) "
338 query=$query" AND ISNULL(fReturnCode)) "
339 fi
340 counter=`echo $counter + 1 | bc -l`
341 done
342 # add condition for the status of the step itself
343 if [ $counter -gt 0 ]
344 then
345 query=$query" AND "
346 fi
347 query=$query" ISNULL("$step"Status.fStartTime) "
348 query=$query" AND ISNULL("$step"Status.fStopTime) "
349 query=$query" AND ISNULL("$step"Status.fReturnCode) "
350}
351
352# function to get todolist
353# returns the next or the list of next steps
354function gettodo()
355{
356 # reset the variable for the number of the next step
357 process=
358 printprocesslog "INFO getting todo..."
359 getstepinfo
360 # get query
361 query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
362 for (( j=1 ; j < ${#prims[@]} ; j++ ))
363 do
364 query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
365 done
366 # get middle par of query
367 middlepartofquery
368 # add requirement for production host in case it is needed
369 if [ "$2 " != " " ]
370 then
371 query=$query" AND fProductionHostKEY=$2 "
372 fi
373 # order by priority to the the number of the next step to be done
374 query=$query" ORDER BY "$step"Status.fPriority desc "
375 # add limitation in case only one or a limited number of
376 # processes should be executed
377 if [ "$1 " != " " ]
378 then
379 query=$query" limit 0, $1 "
380 fi
381 # print query
382 #echo " gettodo QUERY: "$query
383 printprocesslog "INFO gettodo QUERY: "$query
384 # execute query
385 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
386 then
387 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
388 finish
389 fi
390 # get numbers of next step from mysql result
391 if [ "$process" = "" ]
392 then
393 printprocesslog "INFO => nothing to do"
394 finish
395 else
396 primaries=( $process )
397 num=`expr ${#primaries[@]} / ${#prims[@]} `
398 fi
399}
400
401# function to get the number of processes which still have to be done
402function getstatus()
403{
404 # reset the variable for the number of steps to be done
405 numproc=
406 getstepinfo
407 # get query
408 query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
409 # get middle part of query
410 middlepartofquery
411 # add requirement for production host in case it is needed
412 if [ "$1 " != " " ]
413 then
414 query=$query" AND fProductionHostKEY=$1 "
415 fi
416 # group by an 'artifical' column to get the number of lines
417 query=$query" GROUP BY 2 "
418 # printing query
419 #echo " getstatus QUERY: "$query
420 printprocesslog "INFO getstatus QUERY: "$query
421 # execute query
422 if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
423 then
424 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
425 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
426 continue
427 fi
428 # get number of processes from mysql result
429 if [ "${numprocs[1]}" = "" ]
430 then
431 numproc=0
432 else
433 numproc=${numprocs[0]}
434 fi
435}
436
437# function to set status of a process in the db
438function setstatus()
439{
440 # remark:
441 # this function does not include the 'Default' flag
442 # for resetting steps
443
444 # for dowebplots (there are steps which have no entry in the DB)
445 if [ "$step" = "no" ]
446 then
447 return
448 fi
449
450 # reset status values
451 starttime=NULL
452 stoptime=NULL
453 returncode=NULL
454 # evaluate the status values
455 case $@ in
456 start) printprocesslog "INFO setstatus start"
457 starttime="Now()"
458 ;;
459 stop) case $check in
460 ok) printprocesslog "INFO setstatus stop - ok"
461 starttime=noreset
462 stoptime="Now()"
463 ;;
464 no) printprocesslog "INFO setstatus stop - nothing new"
465 check="ok"
466 ;;
467 *) printprocesslog "INFO setstatus stop - failed"
468 starttime=noreset
469 stoptime="Now()"
470 if [ "$check" == "" ]
471 then
472 returncode=1
473 else
474 returncode=$check
475 fi
476 check="ok"
477 ;;
478 esac
479 ;;
480 *) printprocesslog "ERROR function setstatus got wrong variable"
481 finish
482 ;;
483 esac
484 # get
485 getstepinfo
486
487 # get the influences from the steps.rc by evaluating the needs of all steps
488 influences=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
489
490 # get query
491 query=" UPDATE "$step"Status "
492 # add joins to the influenced tables
493 specialinfljoin=`getjoin $step "SpecialInfluence"`
494 if ! [ "$specialinfljoin" = "" ]
495 then
496 query=$query" "$specialinfljoin
497 fi
498 for influence in $influences
499 do
500 infljoin=`getjoin $influence "Influence"`
501 if [ "$infljoin" = "" ]
502 then
503 inflprims=( `getprimary $influence` )
504 query=$query" LEFT JOIN "$influence"Status USING (${inflprims[@]}) "
505 else
506 query=$query" "$infljoin
507 fi
508 done
509 # set the status values according to the new status of the step
510 query=$query" SET "
511 if ! [ "$starttime" = "noreset" ]
512 then
513 query=$query" "$step"Status.fStartTime=$starttime, "
514 fi
515 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
516 # set also the status values of the influenced steps
517 for influence in $influences
518 do
519 query=$query", "$influence"Status.fStartTime=NULL "
520 query=$query", "$influence"Status.fStopTime=NULL "
521 query=$query", "$influence"Status.fReturnCode=NULL "
522 done
523 # give the condition for which step the status values have to be set
524 query=$query" WHERE "
525 if [ "$s" = "" ]
526 then
527 s=0
528 fi
529 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
530 for (( j=1 ; j < ${#prims[@]} ; j++ ))
531 do
532 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
533 done
534 # print query
535 #echo " setstatus QUERY: "$query
536 printprocesslog "INFO setstatus QUERY: "$query
537 # execute query
538 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
539 then
540 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
541 finish
542 fi
543}
544
545# function to send a mysql query
546function sendquery()
547{
548 getdbsetup
549 printprocesslog "INFO sendquery QUERY: "$query
550 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
551 then
552 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
553 return 1
554 fi
555 if [ "$val" = "NULL" ]
556 then
557 val=
558 fi
559 echo $val
560 return 0
561}
562
Note: See TracBrowser for help on using the repository browser.