source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9592

Last change on this file since 9592 was 9592, checked in by Daniela Dorner, 11 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 16.9 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58 if [ ! -d $@ ]
59 then
60 mkdir -pv $@
61 if [ ! -d $@ ]
62 then
63 if ! [ "$processlog" = "" ]
64 then
65 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66 else
67 echo "could not make dir "$@
68 fi
69 if ls $lockfile >/dev/null 2>&1
70 then
71 rm -v $lockfile
72 fi
73 exit
74 fi
75 fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91 makedir $processlogpath
92 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99 then
100 printprocesslog "INFO " `rm -v $lockfile`
101 fi
102 printprocesslog "INFO finished $0"
103 exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110#failed codes
111#sequence build status
112Fbuildsequ=1
113Fdoexcl=2
114#run process status
115Ftimecorr=3
116Ffillraw=4
117Fsinope=5
118Ffillsinope=6
119Fresetexcl=7
120#sequence process status
121Fwritesequfile=8
122Ffilesavail=9
123Fnoccfile=10
124Fnocacofile=11
125Fmerppcc=12
126Fmerppcaco=13
127Fcallisto=14
128Ffillcalib=15
129Ffillsignal=16
130Fstar=17
131Ffillstar=18
132#dataset process status
133Fwritedatasetfile=19
134Fstardone=20
135Fganymed=21
136Ffillganymed=22
137#again run process status
138FCompmux=26
139Fdowebplots=27
140#again mc run process status
141Fmccallisto=28
142Ffillmccalib=29
143Ffillmcsignal=30
144Fmcstar=31
145Ffillmcstar=32
146Fcorsikasimtel=33
147Fchimp=34
148Fchimpcp=35
149Fctastar=36
150Fctastarcp=37
151FctastereoA=38
152FctastereoB=39
153FctastereoC=40
154FctastereoD=41
155FctastereoE=42
156FctastereoF=43
157FctastereoG=44
158FctastereoH=45
159Fctastereocp=46
160
161# setup for jobmanager:
162# log files (can't be defined in script itself, as script can run longer
163# than one day
164jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
165jmscriptlog=$runlogpath/jobmanager`date +%F`.log
166
167# check if rc-files are available
168if ! ls $steps >/dev/null
169then
170 echo "Can't find steps.rc ($steps)"
171 finish
172fi
173if ! ls $sqlrc >/dev/null
174then
175 echo "Can't find sql.rc ($sqlrc)"
176 finish
177fi
178
179# resetting values
180pno=0
181totalpno=0
182running=0
183queued=0
184runningscript=0
185queuedscript=0
186stillinqueue=0
187
188
189# alias (we cannot check the beginning of the line due to
190# color codes in filldotraw.C)
191alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
192
193
194# in the following the functions, which are needed by several scripts, are
195# defined
196
197# function to check if a process is already locked
198# command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202 then
203 printprocesslog "WARN lockfile $lockfile exists"
204 $@
205 exit
206 else
207 printprocesslog "INFO created lockfile $lockfile"
208 fi
209}
210
211# print the current status values
212function printstatusvalues()
213{
214 echo "the current values are:"
215 echo " starttime=$starttime"
216 echo " stoptime=$stoptime"
217 echo " returncode=$returncode"
218 echo "-- check: -$check-"
219 echo ""
220}
221
222# get the db-setup from the sql.rc
223function getdbsetup()
224{
225 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
226 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
227 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
228 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
229# echo "setup: "
230# echo " db: "$db
231# echo " pw: "$pw
232# echo " us: "$us
233# echo " ho: "$ho
234}
235
236# function to get the needed information from the dependencies-file steps.rc
237function getstepinfo()
238{
239 getdbsetup
240 needs=`grep "$step[.]Needs:" $steps | grep -v '#' | sed -e "s/$step[.]Needs://"`
241 noderestricted=`grep "$step[.]NodeRestricted:" $steps | grep -v '#' | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
242 prims=( `grep "$step[.]Primaries:" $steps | grep -v '#' | sed -e "s/$step[.]Primaries://"` )
243# echo " needs: $needs"
244# echo " noderestricted: $noderestricted"
245# echo " prims: ${prims[@]}"
246}
247
248# function to get the primaries of a step from the dependencies-file steps.rc
249function getprimary()
250{
251 getdbsetup
252 grep $@"[.]Primaries:" $steps | grep -v '#' | sed -e "s/$@[.]Primaries://"
253}
254
255# function to get the join of a step
256# (normal join but using different primaries)
257# from the dependencies-file steps.rc
258function getjoin()
259{
260 getdbsetup
261 grep $@"[.]Join:" $steps | grep -v '#' | sed -e "s/$@[.]Join://"
262}
263
264# function to get the special join of a step
265# (i.e. join with a different table)
266# from the dependencies-file steps.rc
267function getspecialjoin()
268{
269 getdbsetup
270 grep $@"[.]SpecialJoin:" $steps | grep -v '#' | sed -e "s/$@[.]SpecialJoin://"
271}
272
273# function to get the needs of a step from the dependencies-file steps.rc
274function getneeds()
275{
276 getdbsetup
277 grep $@"[.]Needs:" $steps | sed -e "s/$@[.]Needs://"
278}
279
280# function to create the middle part of a query
281# which is identical for the functions getstatus() and gettodo()
282function middlepartofquery()
283{
284 # add from which table the information is queried
285 query=$query" FROM "$step"Status "
286 # add the joins to the tables in which the status of the preceding steps is stored
287 for need in $needs
288 do
289 needprims=( `getprimary $need` )
290 # make sure that the correct joins are used
291 # in case the primaries of the step and the need are
292 # the same, the tables are join with these primaries
293 # otherwise it is checked whether there has to be a
294 # join with different primaries (indicated as 'Join'
295 # in the steps.rc
296 # 'SpecialJoins' do not have to be taken into account here
297 if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
298 then
299 query=$query" LEFT JOIN "$need"Status USING (${prims[@]}) "
300 else
301 stdjoin=`getjoin $need`
302 if ! [ "$stdjoin" = "" ]
303 then
304 query=$query" "$stdjoin" "
305 fi
306 fi
307 done
308 # add condition
309 query=$query" WHERE "
310 # add condition for the status of the peceding steps
311 counter=0
312 for need in $needs
313 do
314 if [ $counter -gt 0 ]
315 then
316 query=$query" AND "
317 fi
318 needprims=( `getprimary $need` )
319 # in case the primaries of the tables agree
320 # or there is just a normal 'Join' needed
321 # only the condition is added to the query
322 # for tables which need a join with a different
323 # table, a special query is added using the
324 # 'SpecialJoin' from the steps.rc
325 if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ] || ! [ "$stdjoin" = "" ]
326 then
327 query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
328 query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
329 query=$query" ISNULL("$need"Status.fReturnCode) "
330 else
331 query=$query" (SELECT COUNT(*) FROM "$need"Status "
332 query=$query" "`getspecialjoin $need`" "
333 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
334 for (( j=1 ; j < ${#prims[@]} ; j++ ))
335 do
336 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
337 done
338 query=$query") = (SELECT COUNT(*) FROM "$need"Status "
339 query=$query" "`getspecialjoin $need`" "
340 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
341 for (( j=1 ; j < ${#prims[@]} ; j++ ))
342 do
343 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
344 done
345 query=$query" AND NOT ISNULL(fStartTime) "
346 query=$query" AND NOT ISNULL(fStopTime) "
347 query=$query" AND ISNULL(fReturnCode)) "
348 fi
349 counter=`echo $counter + 1 | bc -l`
350 done
351 # add condition for the status of the step itself
352 if [ $counter -gt 0 ]
353 then
354 query=$query" AND "
355 fi
356 query=$query" ISNULL("$step"Status.fStartTime) "
357 query=$query" AND ISNULL("$step"Status.fStopTime) "
358 query=$query" AND ISNULL("$step"Status.fReturnCode) "
359}
360
361# function to get todolist
362# returns the next or the list of next steps
363function gettodo()
364{
365 # reset the variable for the number of the next step
366 process=
367 printprocesslog "INFO getting todo..."
368 getstepinfo
369 # get query
370 query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
371 for (( j=1 ; j < ${#prims[@]} ; j++ ))
372 do
373 query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
374 done
375 # get middle par of query
376 middlepartofquery
377 # add requirement for production host in case it is needed
378 if [ "$2 " != " " ]
379 then
380 query=$query" AND fProductionHostKEY=$2 "
381 fi
382 # order by priority to the the number of the next step to be done
383 query=$query" ORDER BY "$step"Status.fPriority desc "
384 # add limitation in case only one or a limited number of
385 # processes should be executed
386 if [ "$1 " != " " ]
387 then
388 query=$query" limit 0, $1 "
389 fi
390 # print query
391 #echo " gettodo QUERY: "$query
392 printprocesslog "INFO gettodo QUERY: "$query
393 # execute query
394 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
395 then
396 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
397 finish
398 fi
399 # get numbers of next step from mysql result
400 if [ "$process" = "" ]
401 then
402 printprocesslog "INFO => nothing to do"
403 finish
404 else
405 primaries=( $process )
406 num=`expr ${#primaries[@]} / ${#prims[@]} `
407 fi
408}
409
410# function to get the number of processes which still have to be done
411function getstatus()
412{
413 # reset the variable for the number of steps to be done
414 numproc=
415 getstepinfo
416 # get query
417 query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
418 # get middle part of query
419 middlepartofquery
420 # add requirement for production host in case it is needed
421 if [ "$1 " != " " ]
422 then
423 query=$query" AND fProductionHostKEY=$1 "
424 fi
425 # group by an 'artifical' column to get the number of lines
426 query=$query" GROUP BY 2 "
427 # printing query
428 #echo " getstatus QUERY: "$query
429 printprocesslog "INFO getstatus QUERY: "$query
430 # execute query
431 if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
432 then
433 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
434 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
435 continue
436 fi
437 # get number of processes from mysql result
438 if [ "${numprocs[1]}" = "" ]
439 then
440 numproc=0
441 else
442 numproc=${numprocs[0]}
443 fi
444}
445
446# function to set status of a process in the db
447function setstatus()
448{
449 # remark:
450 # this function does not include the 'Default' flag
451 # for resetting steps
452
453 # for dowebplots (there are steps which have no entry in the DB)
454 if [ "$step" = "no" ]
455 then
456 return
457 fi
458
459 # reset status values
460 starttime=NULL
461 stoptime=NULL
462 returncode=NULL
463 # evaluate the status values
464 case $@ in
465 start) printprocesslog "INFO setstatus start"
466 starttime="Now()"
467 ;;
468 stop) case $check in
469 ok) printprocesslog "INFO setstatus stop - ok"
470 starttime=noreset
471 stoptime="Now()"
472 ;;
473 no) printprocesslog "INFO setstatus stop - nothing new"
474 check="ok"
475 ;;
476 *) printprocesslog "INFO setstatus stop - failed"
477 starttime=noreset
478 stoptime="Now()"
479 if [ "$check" == "" ]
480 then
481 returncode=1
482 else
483 returncode=$check
484 fi
485 check="ok"
486 ;;
487 esac
488 ;;
489 *) printprocesslog "ERROR function setstatus got wrong variable"
490 finish
491 ;;
492 esac
493 # get
494 getstepinfo
495
496 # get the influences from the steps.rc by evaluating the needs of all steps
497 influences=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
498
499 # get query
500 query=" UPDATE "$step"Status "
501 # add joins to the influenced tables
502 for influence in $influences
503 do
504 # make sure that the right join is done
505 # in case the is a 'Join' given for the
506 # influenced step, this is added
507 # otherwise it is checked whether the executed step
508 # itself needs a 'Join' (i.e. has different primaries)
509 # if yes, the tables are joined with the primaries of the influenced step
510 # else with the primaries of the executed step itself
511 stdjoin=`getjoin $influence`
512 if ! [ "$stdjoin" = "" ]
513 then
514 query=$query" "$stdjoin" "
515 else
516 stdjoin2=`getjoin $step`
517 if [ "$stdjoin2" == "" ]
518 then
519 query=$query" LEFT JOIN "$influence"Status USING("`getprimary $influence`") "
520 else
521 query=$query" LEFT JOIN "$influence"Status USING("`getprimary $step`") "
522 fi
523 fi
524 done
525 # set the status values according to the new status of the step
526 query=$query" SET "
527 if ! [ "$starttime" = "noreset" ]
528 then
529 query=$query" "$step"Status.fStartTime=$starttime, "
530 fi
531 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
532 # set also the status values of the influenced steps
533 for influence in $influences
534 do
535 query=$query", "$influence"Status.fStartTime=NULL "
536 query=$query", "$influence"Status.fStopTime=NULL "
537 query=$query", "$influence"Status.fReturnCode=NULL "
538 done
539 # give the condition for which step the status values have to be set
540 query=$query" WHERE "
541 if [ "$s" = "" ]
542 then
543 s=0
544 fi
545 query=$query" "`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
546 for (( j=1 ; j < ${#prims[@]} ; j++ ))
547 do
548 query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
549 done
550 # print query
551 #echo " setstatus QUERY: "$query
552 printprocesslog "INFO setstatus QUERY: "$query
553 # execute query
554 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
555 then
556 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
557 finish
558 fi
559}
560
561# function to send a mysql query
562function sendquery()
563{
564 getdbsetup
565 printprocesslog "INFO sendquery QUERY: "$query
566 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
567 then
568 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
569 return 1
570 fi
571 if [ "$val" = "NULL" ]
572 then
573 val=
574 fi
575 echo $val
576 return 0
577}
578
Note: See TracBrowser for help on using the repository browser.