source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9585

Last change on this file since 9585 was 9585, checked in by Daniela Dorner, 15 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 15.0 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58 if [ ! -d $@ ]
59 then
60 mkdir -pv $@
61 if [ ! -d $@ ]
62 then
63 if ! [ "$processlog" = "" ]
64 then
65 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66 else
67 echo "could not make dir "$@
68 fi
69 if ls $lockfile >/dev/null 2>&1
70 then
71 rm -v $lockfile
72 fi
73 exit
74 fi
75 fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91 makedir $processlogpath
92 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99 then
100 printprocesslog "INFO " `rm -v $lockfile`
101 fi
102 printprocesslog "INFO finished $0"
103 exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110#failed codes
111#sequence build status
112Fbuildsequ=1
113Fdoexcl=2
114#run process status
115Ftimecorr=3
116Ffillraw=4
117Fsinope=5
118Ffillsinope=6
119Fresetexcl=7
120#sequence process status
121Fwritesequfile=8
122Ffilesavail=9
123Fnoccfile=10
124Fnocacofile=11
125Fmerppcc=12
126Fmerppcaco=13
127Fcallisto=14
128Ffillcalib=15
129Ffillsignal=16
130Fstar=17
131Ffillstar=18
132#dataset process status
133Fwritedatasetfile=19
134Fstardone=20
135Fganymed=21
136Ffillganymed=22
137#again run process status
138FCompmux=26
139Fdowebplots=27
140#again mc run process status
141Fmccallisto=28
142Ffillmccalib=29
143Ffillmcsignal=30
144Fmcstar=31
145Ffillmcstar=32
146Fcorsikasimtel=33
147Fchimp=34
148Fchimpcp=35
149Fctastar=36
150Fctastarcp=37
151FctastereoA=38
152FctastereoB=39
153FctastereoC=40
154FctastereoD=41
155FctastereoE=42
156FctastereoF=43
157FctastereoG=44
158FctastereoH=45
159Fctastereocp=46
160
161# setup for jobmanager:
162# log files (can't be defined in script itself, as script can run longer
163# than one day
164jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
165jmscriptlog=$runlogpath/jobmanager`date +%F`.log
166
167# check if rc-files are available
168if ! ls $steps >/dev/null
169then
170 echo "Can't find steps.rc ($steps)"
171 finish
172fi
173if ! ls $sqlrc >/dev/null
174then
175 echo "Can't find sql.rc ($sqlrc)"
176 finish
177fi
178
179# resetting values
180pno=0
181totalpno=0
182running=0
183queued=0
184runningscript=0
185queuedscript=0
186stillinqueue=0
187
188
189# alias (we cannot check the beginning of the line due to
190# color codes in filldotraw.C)
191alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
192
193
194# in the following the functions, which are needed by several scripts, are
195# defined
196
197# function to check if a process is already locked
198# command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202 then
203 printprocesslog "WARN lockfile $lockfile exists"
204 $@
205 exit
206 else
207 printprocesslog "INFO created lockfile $lockfile"
208 fi
209}
210
211# print the current status values
212function printstatusvalues()
213{
214 echo "the current values are:"
215 echo " starttime=$starttime"
216 echo " stoptime=$stoptime"
217 echo " returncode=$returncode"
218 echo "-- check: -$check-"
219 echo ""
220}
221
222# get the db-setup from the sql.rc
223function getdbsetup()
224{
225 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
226 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
227 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
228 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
229# echo "setup: "
230# echo " db: "$db
231# echo " pw: "$pw
232# echo " us: "$us
233# echo " ho: "$ho
234}
235
236# function to get the needed information from the dependencies-file steps.rc
237function getstepinfo()
238{
239 getdbsetup
240 needs=`grep "$step[.]Needs:" $steps | sed -e "s/$step[.]Needs://"`
241 noderestricted=`grep "$step[.]NodeRestricted:" $steps | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
242 prims=( `grep "$step[.]Primaries:" $steps | sed -e "s/$step[.]Primaries://"` )
243# echo " needs: $needs"
244# echo " noderestricted: $noderestricted"
245# echo " prims: ${prims[@]}"
246}
247
248# function to get the primaries of a step from the dependencies-file steps.rc
249function getprimary()
250{
251 getdbsetup
252 grep $@"[.]Primaries:" $steps | sed -e "s/$@[.]Primaries://"
253}
254
255# function to get the join of a step from the dependencies-file steps.rc
256function getjoin()
257{
258 getdbsetup
259 grep $@"[.]Join:" $steps | sed -e "s/$@[.]Join://"
260}
261
262# function to create the middle part of a query
263# which is identical for the functions getstatus() and gettodo()
264function middlepartofquery()
265{
266 # add from which table the information is queried
267 query=$query" FROM "$step"Status "
268 # add the joins to the tables in which the status of the preceding steps is stored
269 for need in $needs
270 do
271 needprims=( `getprimary $need` )
272 if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
273 then
274 query=$query" LEFT JOIN "$need"Status USING (${prims[@]}) "
275 fi
276 done
277 # add condition
278 query=$query" WHERE "
279 # add condition for the status of the peceding steps
280 counter=0
281 for need in $needs
282 do
283 if [ $counter -gt 0 ]
284 then
285 query=$query" AND "
286 fi
287 needprims=( `getprimary $need` )
288 # in case the primaries of the tables agree
289 # only the condition is given
290 # for tables with differing primaries a special query
291 # is needed
292 if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
293 then
294 query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
295 query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
296 query=$query" ISNULL("$need"Status.fReturnCode) "
297 else
298 query=$query" (SELECT COUNT(*) FROM "$need"Status "
299 query=$query" "`getjoin $need`" "
300 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
301 for (( j=1 ; j < ${#prims[@]} ; j++ ))
302 do
303 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
304 done
305 query=$query") = (SELECT COUNT(*) FROM "$need"Status "
306 query=$query" "`getjoin $need`" "
307 query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
308 for (( j=1 ; j < ${#prims[@]} ; j++ ))
309 do
310 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
311 done
312 query=$query" AND NOT ISNULL(fStartTime) "
313 query=$query" AND NOT ISNULL(fStopTime) "
314 query=$query" AND ISNULL(fReturnCode)) "
315 fi
316 counter=`echo $counter + 1 | bc -l`
317 done
318 # add condition for the status of the step itself
319 query=$query" AND ISNULL("$step"Status.fStartTime) "
320 query=$query" AND ISNULL("$step"Status.fStopTime) "
321 query=$query" AND ISNULL("$step"Status.fReturnCode) "
322}
323
324# function to get todolist
325# returns the next or the list of next steps
326function gettodo()
327{
328 # reset the variable for the number of the next step
329 process=
330 printprocesslog "INFO getting todo..."
331 getstepinfo
332 # get query
333 query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
334 for (( j=1 ; j < ${#prims[@]} ; j++ ))
335 do
336 query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
337 done
338 # get middle par of query
339 middlepartofquery
340 # add requirement for production host in case it is needed
341 if [ "$2 " != " " ]
342 then
343 query=$query" AND fProductionHostKEY=$2 "
344 fi
345 # order by priority to the the number of the next step to be done
346 query=$query" ORDER BY "$step"Status.fPriority desc "
347 # add limitation in case only one or a limited number of
348 # processes should be executed
349 if [ "$1 " != " " ]
350 then
351 query=$query" limit 0, $1 "
352 fi
353 # print query
354 echo " gettodo QUERY: "$query
355 printprocesslog "INFO gettodo QUERY: "$query
356 # execute query
357 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
358 then
359 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
360 finish
361 fi
362 # get numbers of next step from mysql result
363 if [ "$process" = "" ]
364 then
365 printprocesslog "INFO => nothing to do"
366 finish
367 else
368 primaries=( $process )
369 num=`expr ${#primaries[@]} / ${#prims[@]} `
370 fi
371}
372
373# function to get the number of processes which still have to be done
374function getstatus()
375{
376 # reset the variable for the number of steps to be done
377 numproc=
378 getstepinfo
379 # get query
380 query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
381 # get middle part of query
382 middlepartofquery
383 # add requirement for production host in case it is needed
384 if [ "$1 " != " " ]
385 then
386 query=$query" AND fProductionHostKEY=$1 "
387 fi
388 # group by an 'artifical' column to get the number of lines
389 query=$query" GROUP BY 2 "
390 # printing query
391 echo " getstatus QUERY: "$query
392 printprocesslog "INFO getstatus QUERY: "$query
393 # execute query
394 if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
395 then
396 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
397 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
398 continue
399 fi
400 # get number of processes from mysql result
401 if [ "${numprocs[1]}" = "" ]
402 then
403 numproc=0
404 else
405 numproc=${numprocs[0]}
406 fi
407}
408
409# function to set status of a process in the db
410function setstatus()
411{
412 # remark:
413 # this function does not include the 'Default' flag
414 # for resetting steps
415
416 # for dowebplots (there are steps which have no entry in the DB)
417 if [ "$step" = "no" ]
418 then
419 return
420 fi
421
422 # reset status values
423 starttime=NULL
424 stoptime=NULL
425 returncode=NULL
426 # evaluate the status values
427 case $@ in
428 start) printprocesslog "INFO setstatus start"
429 starttime="Now()"
430 ;;
431 stop) case $check in
432 ok) printprocesslog "INFO setstatus stop - ok"
433 starttime=noreset
434 stoptime="Now()"
435 ;;
436 no) printprocesslog "INFO setstatus stop - nothing new"
437 check="ok"
438 ;;
439 *) printprocesslog "INFO setstatus stop - failed"
440 starttime=noreset
441 stoptime="Now()"
442 if [ "$check" == "" ]
443 then
444 returncode=1
445 else
446 returncode=$check
447 fi
448 check="ok"
449 ;;
450 esac
451 ;;
452 *) printprocesslog "ERROR function setstatus got wrong variable"
453 finish
454 ;;
455 esac
456 # get
457 getstepinfo
458
459 # get the influences from the steps.rc by evaluating the needs of all steps
460 influences=`grep $step $rc | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
461
462 # get query
463 query=" UPDATE "$step"Status "
464 # add joins to the influenced tables
465 for influence in $influences
466 do
467 query=$query" LEFT JOIN $influence USING("`getprimary $influence`") "
468 specialjoin=`getjoin $influence`
469 if ! [ "$specialjoin" = "" ]
470 then
471 query=$query$specialjoin
472 fi
473 done
474 # set the status values according to the new status of the step
475 query=$query" SET "
476 if ! [ "$starttime" = "noreset" ]
477 then
478 query=$query" "$step"Status.fStartTime=$starttime, "
479 fi
480 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
481 # set also the status values of the influenced steps
482 for influence in $influences
483 do
484 query=$query", "$influence"Status.fStartTime=NULL "
485 query=$query", "$influence"Status.fStopTime=NULL "
486 query=$query", "$influence"Status.fReturnCode=NULL "
487 done
488 # give the condition for which step the status values have to be set
489 query=$query" WHERE "
490 if [ "$s" = "" ]
491 then
492 s=0
493 fi
494 query=$query" "`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
495 for (( j=1 ; j < ${#prims[@]} ; j++ ))
496 do
497 query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
498 done
499 # print query
500 echo " setstatus QUERY: "$query
501 printprocesslog "INFO setstatus QUERY: "$query
502 # execute query
503 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
504 then
505 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
506 finish
507 fi
508}
509
510# function to send a mysql query
511function sendquery()
512{
513 getdbsetup
514 printprocesslog "INFO sendquery QUERY: "$query
515 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
516 then
517 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
518 return 1
519 fi
520 if [ "$val" = "NULL" ]
521 then
522 val=
523 fi
524 echo $val
525 return 0
526}
527
Note: See TracBrowser for help on using the repository browser.