source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9456

Last change on this file since 9456 was 9456, checked in by Daniela Dorner, 15 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 11.1 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2007
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40source `dirname $0`/setup
41
42mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'`
43macrospath=$mars/datacenter/macros
44scriptspath=$mars/datacenter/scripts
45
46datetime=`date +%F-%H-%M-%S`
47
48
49# function to make sure that a directory is made
50function makedir()
51{
52 if [ ! -d $@ ]
53 then
54 mkdir -pv $@
55 if [ ! -d $@ ]
56 then
57 echo "could not make dir "$@
58 if ! [ "$processlog" = "" ]
59 then
60 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] could not make dir "$@ >> $processlog
61 fi
62 if ls $lockfile >/dev/null 2>&1
63 then
64 rm -v $lockfile
65 fi
66 exit
67 fi
68 fi
69}
70
71# logging paths for runlogs and processlog
72runlogpath=$logpath/run/`date +%Y/%m/%d`
73processlogpath=$logpath/processlog
74makedir $runlogpath
75makedir $processlogpath
76processlog=$processlogpath/process`date +%F`.log
77
78makedir $lockpath
79
80
81# function to provide proper logging in a single logfile ($processlog)
82function printprocesslog
83{
84 makedir $processlogpath
85 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
86}
87
88# function to exit a script properly
89function finish()
90{
91 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
92 then
93 printprocesslog "INFO " `rm -v $lockfile`
94 fi
95 printprocesslog "INFO finished $0"
96 exit
97}
98
99
100# set checkvalue to ok at the beginning of the scripts
101check="ok"
102
103#failed codes
104#sequence build status
105Fbuildsequ=1
106Fdoexcl=2
107#run process status
108Ftimecorr=3
109Ffillraw=4
110Fsinope=5
111Ffillsinope=6
112Fresetexcl=7
113#sequence process status
114Fwritesequfile=8
115Ffilesavail=9
116Fnoccfile=10
117Fnocacofile=11
118Fmerppcc=12
119Fmerppcaco=13
120Fcallisto=14
121Ffillcalib=15
122Ffillsignal=16
123Fstar=17
124Ffillstar=18
125#dataset process status
126Fwritedatasetfile=19
127Fstardone=20
128Fganymed=21
129Ffillganymed=22
130#again run process status
131FCompmux=26
132Fdowebplots=27
133#again mc run process status
134Fmccallisto=28
135Ffillmccalib=29
136Ffillmcsignal=30
137Fmcstar=31
138Ffillmcstar=32
139Fcorsikasimtel=33
140Fchimp=34
141Fchimpcp=35
142Fctastar=36
143Fctastarcp=37
144FctastereoA=38
145FctastereoB=39
146FctastereoC=40
147FctastereoD=41
148FctastereoE=42
149FctastereoF=43
150FctastereoG=44
151FctastereoH=45
152Fctastereocp=46
153
154# setup for jobmanager:
155# log files (can't be defined in script itself, as script can run longer
156# than one day
157jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
158jmscriptlog=$runlogpath/jobmanager`date +%F`.log
159steps=$mars/resources/steps.rc
160if ! ls $steps >/dev/null
161then
162 echo "Can't find steps.rc ($steps)"
163 finish
164fi
165sqlrc=$mars/sql.rc
166if ! ls $sqlrc >/dev/null
167then
168 echo "Can't find sql.rc ($sqlrc)"
169 finish
170fi
171# resetting values
172pno=0
173totalpno=0
174running=0
175queued=0
176runningscript=0
177queuedscript=0
178stillinqueue=0
179
180
181# alias (we cannot check the beginning of the line due to
182# color codes in filldotraw.C)
183alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
184
185
186# in the following the functions, which are needed by several scripts, are
187# defined
188
189# function to do continue in a loop and produce according logging
190function cont()
191{
192 date
193 echo ""
194 continue
195}
196
197# function to check if a process is already locked
198# command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202 then
203 printprocesslog "WARN lockfile $lockfile exists"
204 $@
205 exit
206 else
207 printprocesslog "INFO created lockfile $lockfile"
208 fi
209}
210
211function resetstatusvalues()
212{
213 statustime=NULL
214 starttime=NULL
215 returncode=NULL
216 programid=NULL
217 failedtime=NULL
218}
219
220function printstatusvalues()
221{
222 echo "the current values are:"
223 echo " statustime=$statustime"
224 echo " starttime=$starttime"
225 echo " returncode=$returncode"
226 echo " programid=$programid"
227 echo " failedtime=$failedtime"
228 echo "-- check: -$check-"
229 echo ""
230}
231
232# function evaluating the statusvalues
233function evalstatus()
234{
235 case $@ in
236 start) printprocesslog "INFO setstatus start"
237 starttime="Now()"
238 ;;
239 stop) case $check in
240 ok) printprocesslog "INFO setstatus stop - ok"
241 statustime="Now()"
242 ;;
243 no) printprocesslog "INFO setstatus stop - nothing new"
244 check="ok"
245 ;;
246 *) printprocesslog "INFO setstatus stop - failed"
247 starttime=noreset
248 if [ "$check" == "" ]
249 then
250 returncode=1
251 else
252 returncode=$check
253 fi
254 programid=$com
255 failedtime="Now()"
256 check="ok"
257 ;;
258 esac
259 ;;
260 *) printprocesslog "ERROR function evalstatus got wrong variable"
261 finish
262 ;;
263 esac
264}
265
266function getdbsetup()
267{
268 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
269 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
270 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
271 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
272# echo "setup: "
273# echo " db: "$db
274# echo " pw: "$pw
275# echo " us: "$us
276# echo " ho: "$ho
277}
278
279function getstepinfo()
280{
281 table=`grep "$column:" $steps | sed -e "s/[.]$column://" -e 's/#//' -e 's/ //g'`
282 coltab=`grep "$column:" $steps | sed -e 's/://' -e 's/#//' -e 's/ //g'`
283 needs=`grep "$coltab[.]Needs:" $steps | sed -e "s/$coltab[.]Needs://"`
284 influences=`grep "$coltab[.]Influences:" $steps | sed -e "s/$coltab[.]Influences://"`
285 prims=( `grep "$table[.]Primary:" $steps | sed -e "s/$table[.]Primary://"` )
286# echo " column $column - table $table - coltab $coltab"
287# echo " needs: $needs"
288# echo " influences: $influences"
289# echo " prims: ${prims[@]}"
290}
291
292# function to get todolist
293function gettodo()
294{
295 process=
296 printprocesslog "INFO getting todo..."
297 getdbsetup
298 getstepinfo
299 # get query
300 query=" SELECT "${prims[0]}
301 for (( i=1 ; i < ${#prims[@]} ; i++ ))
302 do
303 query=$query", ${prims[$i]}"
304 done
305 query=$query" FROM $table WHERE "
306 if ! echo $needs | grep '#' > /dev/null
307 then
308 for need in $needs
309 do
310 query=$query" NOT ISNULL($need) AND"
311 done
312 fi
313 query=$query" ISNULL($column) "
314 if [ "$2 " != " " ]
315 then
316 query=$query" AND fProductionHostKEY=$2 "
317 fi
318 query=$query" AND ISNULL(fStartTime) AND ISNULL(fFailedTime) AND ISNULL(fProgramId) AND ISNULL(fReturnCode) "
319 query=$query" ORDER BY fPriority desc "
320 if [ "$1 " != " " ]
321 then
322 query=$query" limit 0, $1 "
323 fi
324# echo " QUERY: "$query
325 printprocesslog "INFO gettodo QUERY: "$query
326 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
327 then
328 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
329 finish
330 fi
331
332 if [ "$process" = "" ]
333 then
334 printprocesslog "INFO => nothing to do"
335 finish
336 else
337 primaries=( $process )
338 num=`expr ${#primaries[@]} / ${#prims[@]} `
339 fi
340}
341
342
343# function to get the number of processes which still have to be done
344function getstatus()
345{
346 numproc=
347 getdbsetup
348 getstepinfo
349 # get query
350 query=" SELECT COUNT(*) FROM $table WHERE "
351 if ! echo $needs | grep '#' > /dev/null
352 then
353 for need in $needs
354 do
355 query=$query" NOT ISNULL($need) AND"
356 done
357 fi
358 query=$query" ISNULL($column) "
359 if [ "$1 " != " " ]
360 then
361 query=$query" AND fProductionHostKEY=$1 "
362 fi
363 query=$query" AND ISNULL(fStartTime) AND ISNULL(fFailedTime) AND ISNULL(fProgramId) AND ISNULL(fReturnCode) "
364 query=$query" GROUP BY $column "
365# echo "QUERY: "$query
366 printprocesslog "INFO getstatus QUERY: "$query
367 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
368 then
369 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
370 cont
371 fi
372}
373
374# function to set status of a process in the db
375function setstatus()
376{
377 if [ "$column" = "no" ]
378 then
379 return
380 fi
381 resetstatusvalues
382 evalstatus $@
383 getdbsetup
384 getstepinfo
385 # get query
386 reset=`grep "$coltab[.]Reset:" $steps | sed -e "s/$coltab[.]Reset://" -e 's/ //g'`
387 if [ "$reset" = "no" ]
388 then
389 printprocesslog "ERROR You cannot reset $column for ${primaries[$s+$s]}"
390 finish
391 fi
392 query=" update $table set $column=$statustime"
393 if ! echo $influences | grep '#' > /dev/null
394 then
395 for influence in $influences
396 do
397 query=$query", $influence=NULL"
398 done
399 fi
400 if ! [ "$starttime" = "noreset" ]
401 then
402 query=$query", fStartTime=$starttime"
403 fi
404 query=$query", fFailedTime=$failedtime, fProgramId=$programid, fReturnCode=$returncode "
405 query=$query" where "
406 if [ "$s" = "" ]
407 then
408 s=0
409 fi
410 query=$query" ${prims[0]}='${primaries[$s*${#prims[@]}]}'"
411 for (( i=1 ; i < ${#prims[@]} ; i++ ))
412 do
413 query=$query" AND ${prims[$i]}='${primaries[$s*${#prims[@]}+$i]}' "
414 done
415# echo " QUERY: "$query
416 printprocesslog "INFO setstatus QUERY: "$query
417 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
418 then
419 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
420 finish
421 fi
422
423}
424
425# function to send a mysql query
426function sendquery()
427{
428 getdbsetup
429 printprocesslog "INFO sendquery QUERY: "$query
430 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
431 then
432 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
433 return 1
434 fi
435 if [ "$val" = "NULL" ]
436 then
437 val=
438 fi
439 echo $val
440 return 0
441}
442
Note: See TracBrowser for help on using the repository browser.