source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9586

Last change on this file since 9586 was 9586, checked in by Daniela Dorner, 15 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 15.0 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58 if [ ! -d $@ ]
59 then
60 mkdir -pv $@
61 if [ ! -d $@ ]
62 then
63 if ! [ "$processlog" = "" ]
64 then
65 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66 else
67 echo "could not make dir "$@
68 fi
69 if ls $lockfile >/dev/null 2>&1
70 then
71 rm -v $lockfile
72 fi
73 exit
74 fi
75 fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91 makedir $processlogpath
92 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99 then
100 printprocesslog "INFO " `rm -v $lockfile`
101 fi
102 printprocesslog "INFO finished $0"
103 exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110#failed codes
111#sequence build status
112Fbuildsequ=1
113Fdoexcl=2
114#run process status
115Ftimecorr=3
116Ffillraw=4
117Fsinope=5
118Ffillsinope=6
119Fresetexcl=7
120#sequence process status
121Fwritesequfile=8
122Ffilesavail=9
123Fnoccfile=10
124Fnocacofile=11
125Fmerppcc=12
126Fmerppcaco=13
127Fcallisto=14
128Ffillcalib=15
129Ffillsignal=16
130Fstar=17
131Ffillstar=18
132#dataset process status
133Fwritedatasetfile=19
134Fstardone=20
135Fganymed=21
136Ffillganymed=22
137#again run process status
138FCompmux=26
139Fdowebplots=27
140#again mc run process status
141Fmccallisto=28
142Ffillmccalib=29
143Ffillmcsignal=30
144Fmcstar=31
145Ffillmcstar=32
146Fcorsikasimtel=33
147Fchimp=34
148Fchimpcp=35
149Fctastar=36
150Fctastarcp=37
151FctastereoA=38
152FctastereoB=39
153FctastereoC=40
154FctastereoD=41
155FctastereoE=42
156FctastereoF=43
157FctastereoG=44
158FctastereoH=45
159Fctastereocp=46
160
161# setup for jobmanager:
162# log files (can't be defined in script itself, as script can run longer
163# than one day
164jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
165jmscriptlog=$runlogpath/jobmanager`date +%F`.log
166
167# check if rc-files are available
168if ! ls $steps >/dev/null
169then
170 echo "Can't find steps.rc ($steps)"
171 finish
172fi
173if ! ls $sqlrc >/dev/null
174then
175 echo "Can't find sql.rc ($sqlrc)"
176 finish
177fi
178
179# resetting values
180pno=0
181totalpno=0
182running=0
183queued=0
184runningscript=0
185queuedscript=0
186stillinqueue=0
187
188
189# alias (we cannot check the beginning of the line due to
190# color codes in filldotraw.C)
191alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
192
193
194# in the following the functions, which are needed by several scripts, are
195# defined
196
197# function to check if a process is already locked
198# command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202 then
203 printprocesslog "WARN lockfile $lockfile exists"
204 $@
205 exit
206 else
207 printprocesslog "INFO created lockfile $lockfile"
208 fi
209}
210
211# print the current status values
212function printstatusvalues()
213{
214 echo "the current values are:"
215 echo " starttime=$starttime"
216 echo " stoptime=$stoptime"
217 echo " returncode=$returncode"
218 echo "-- check: -$check-"
219 echo ""
220}
221
222# get the db-setup from the sql.rc
223function getdbsetup()
224{
225 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
226 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
227 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
228 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
229# echo "setup: "
230# echo " db: "$db
231# echo " pw: "$pw
232# echo " us: "$us
233# echo " ho: "$ho
234}
235
236# function to get the needed information from the dependencies-file steps.rc
237function getstepinfo()
238{
239 getdbsetup
240 needs=`grep "$step[.]Needs:" $steps | sed -e "s/$step[.]Needs://"`
241 noderestricted=`grep "$step[.]NodeRestricted:" $steps | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
242 prims=( `grep "$step[.]Primaries:" $steps | sed -e "s/$step[.]Primaries://"` )
243# echo " needs: $needs"
244# echo " noderestricted: $noderestricted"
245# echo " prims: ${prims[@]}"
246}
247
248# function to get the primaries of a step from the dependencies-file steps.rc
249function getprimary()
250{
251 getdbsetup
252 grep $@"[.]Primaries:" $steps | sed -e "s/$@[.]Primaries://"
253}
254
255# function to get the join of a step from the dependencies-file steps.rc
256function getjoin()
257{
258 getdbsetup
259 grep $@"[.]Join:" $steps | sed -e "s/$@[.]Join://"
260}
261
262# function to create the middle part of a query
263# which is identical for the functions getstatus() and gettodo()
264function middlepartofquery()
265{
266 # add from which table the information is queried
267 query=$query" FROM "$step"Status "
268 # add the joins to the tables in which the status of the preceding steps is stored
269 for need in $needs
270 do
271 needprims=( `getprimary $need` )
272 if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
273 then
274 query=$query" LEFT JOIN "$need"Status USING (${prims[@]}) "
275 fi
276 done
277 # add condition
278 query=$query" WHERE "
279 # add condition for the status of the peceding steps
280 counter=0
281 for need in $needs
282 do
283 if [ $counter -gt 0 ]
284 then
285 query=$query" AND "
286 fi
287 needprims=( `getprimary $need` )
288 # in case the primaries of the tables agree
289 # only the condition is given
290 # for tables with differing primaries a special query
291 # is needed
292 if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
293 then
294 query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
295 query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
296 query=$query" ISNULL("$need"Status.fReturnCode) "
297 else
298 query=$query" (SELECT COUNT(*) FROM "$need"Status "
299 query=$query" "`getjoin $need`" "
300 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
301 for (( j=1 ; j < ${#prims[@]} ; j++ ))
302 do
303 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
304 done
305 query=$query") = (SELECT COUNT(*) FROM "$need"Status "
306 query=$query" "`getjoin $need`" "
307 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
308 for (( j=1 ; j < ${#prims[@]} ; j++ ))
309 do
310 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
311 done
312 query=$query" AND NOT ISNULL(fStartTime) "
313 query=$query" AND NOT ISNULL(fStopTime) "
314 query=$query" AND ISNULL(fReturnCode)) "
315 fi
316 counter=`echo $counter + 1 | bc -l`
317 done
318 # add condition for the status of the step itself
319 if [ $counter -gt 0 ]
320 then
321 query=$query" AND "
322 fi
323 query=$query" ISNULL("$step"Status.fStartTime) "
324 query=$query" AND ISNULL("$step"Status.fStopTime) "
325 query=$query" AND ISNULL("$step"Status.fReturnCode) "
326}
327
328# function to get todolist
329# returns the next or the list of next steps
330function gettodo()
331{
332 # reset the variable for the number of the next step
333 process=
334 printprocesslog "INFO getting todo..."
335 getstepinfo
336 # get query
337 query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
338 for (( j=1 ; j < ${#prims[@]} ; j++ ))
339 do
340 query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
341 done
342 # get middle par of query
343 middlepartofquery
344 # add requirement for production host in case it is needed
345 if [ "$2 " != " " ]
346 then
347 query=$query" AND fProductionHostKEY=$2 "
348 fi
349 # order by priority to the the number of the next step to be done
350 query=$query" ORDER BY "$step"Status.fPriority desc "
351 # add limitation in case only one or a limited number of
352 # processes should be executed
353 if [ "$1 " != " " ]
354 then
355 query=$query" limit 0, $1 "
356 fi
357 # print query
358 echo " gettodo QUERY: "$query
359 printprocesslog "INFO gettodo QUERY: "$query
360 # execute query
361 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
362 then
363 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
364 finish
365 fi
366 # get numbers of next step from mysql result
367 if [ "$process" = "" ]
368 then
369 printprocesslog "INFO => nothing to do"
370 finish
371 else
372 primaries=( $process )
373 num=`expr ${#primaries[@]} / ${#prims[@]} `
374 fi
375}
376
377# function to get the number of processes which still have to be done
378function getstatus()
379{
380 # reset the variable for the number of steps to be done
381 numproc=
382 getstepinfo
383 # get query
384 query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
385 # get middle part of query
386 middlepartofquery
387 # add requirement for production host in case it is needed
388 if [ "$1 " != " " ]
389 then
390 query=$query" AND fProductionHostKEY=$1 "
391 fi
392 # group by an 'artifical' column to get the number of lines
393 query=$query" GROUP BY 2 "
394 # printing query
395 echo " getstatus QUERY: "$query
396 printprocesslog "INFO getstatus QUERY: "$query
397 # execute query
398 if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
399 then
400 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
401 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
402 continue
403 fi
404 # get number of processes from mysql result
405 if [ "${numprocs[1]}" = "" ]
406 then
407 numproc=0
408 else
409 numproc=${numprocs[0]}
410 fi
411}
412
413# function to set status of a process in the db
414function setstatus()
415{
416 # remark:
417 # this function does not include the 'Default' flag
418 # for resetting steps
419
420 # for dowebplots (there are steps which have no entry in the DB)
421 if [ "$step" = "no" ]
422 then
423 return
424 fi
425
426 # reset status values
427 starttime=NULL
428 stoptime=NULL
429 returncode=NULL
430 # evaluate the status values
431 case $@ in
432 start) printprocesslog "INFO setstatus start"
433 starttime="Now()"
434 ;;
435 stop) case $check in
436 ok) printprocesslog "INFO setstatus stop - ok"
437 starttime=noreset
438 stoptime="Now()"
439 ;;
440 no) printprocesslog "INFO setstatus stop - nothing new"
441 check="ok"
442 ;;
443 *) printprocesslog "INFO setstatus stop - failed"
444 starttime=noreset
445 stoptime="Now()"
446 if [ "$check" == "" ]
447 then
448 returncode=1
449 else
450 returncode=$check
451 fi
452 check="ok"
453 ;;
454 esac
455 ;;
456 *) printprocesslog "ERROR function setstatus got wrong variable"
457 finish
458 ;;
459 esac
460 # get
461 getstepinfo
462
463 # get the influences from the steps.rc by evaluating the needs of all steps
464 influences=`grep $step $rc | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
465
466 # get query
467 query=" UPDATE "$step"Status "
468 # add joins to the influenced tables
469 for influence in $influences
470 do
471 query=$query" LEFT JOIN $influence USING("`getprimary $influence`") "
472 specialjoin=`getjoin $influence`
473 if ! [ "$specialjoin" = "" ]
474 then
475 query=$query$specialjoin
476 fi
477 done
478 # set the status values according to the new status of the step
479 query=$query" SET "
480 if ! [ "$starttime" = "noreset" ]
481 then
482 query=$query" "$step"Status.fStartTime=$starttime, "
483 fi
484 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
485 # set also the status values of the influenced steps
486 for influence in $influences
487 do
488 query=$query", "$influence"Status.fStartTime=NULL "
489 query=$query", "$influence"Status.fStopTime=NULL "
490 query=$query", "$influence"Status.fReturnCode=NULL "
491 done
492 # give the condition for which step the status values have to be set
493 query=$query" WHERE "
494 if [ "$s" = "" ]
495 then
496 s=0
497 fi
498 query=$query" "`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
499 for (( j=1 ; j < ${#prims[@]} ; j++ ))
500 do
501 query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
502 done
503 # print query
504 echo " setstatus QUERY: "$query
505 printprocesslog "INFO setstatus QUERY: "$query
506 # execute query
507 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
508 then
509 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
510 finish
511 fi
512}
513
514# function to send a mysql query
515function sendquery()
516{
517 getdbsetup
518 printprocesslog "INFO sendquery QUERY: "$query
519 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
520 then
521 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
522 return 1
523 fi
524 if [ "$val" = "NULL" ]
525 then
526 val=
527 fi
528 echo $val
529 return 0
530}
531
Note: See TracBrowser for help on using the repository browser.