source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 9489

Last change on this file since 9489 was 9489, checked in by Daniela Dorner, 15 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 11.1 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2007
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then
36 echo "Please start your script with an absolute path."
37 exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then
42 echo "Please set the environment variable \$AUTOMATIONSETUP."
43 exit
44fi
45
46source `dirname $0`/setup.$AUTOMATIONSETUP
47
48datetime=`date +%F-%H-%M-%S`
49
50
51# function to make sure that a directory is made
52function makedir()
53{
54 if [ ! -d $@ ]
55 then
56 mkdir -pv $@
57 if [ ! -d $@ ]
58 then
59 echo "could not make dir "$@
60 if ! [ "$processlog" = "" ]
61 then
62 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] could not make dir "$@ >> $processlog
63 fi
64 if ls $lockfile >/dev/null 2>&1
65 then
66 rm -v $lockfile
67 fi
68 exit
69 fi
70 fi
71}
72
73# logging paths for runlogs and processlog
74runlogpath=$logpath/run/`date +%Y/%m/%d`
75processlogpath=$logpath/processlog
76makedir $runlogpath
77makedir $processlogpath
78processlog=$processlogpath/process`date +%F`.log
79
80makedir $lockpath
81
82
83# function to provide proper logging in a single logfile ($processlog)
84function printprocesslog
85{
86 makedir $processlogpath
87 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
88}
89
90# function to exit a script properly
91function finish()
92{
93 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
94 then
95 printprocesslog "INFO " `rm -v $lockfile`
96 fi
97 printprocesslog "INFO finished $0"
98 exit
99}
100
101
102# set checkvalue to ok at the beginning of the scripts
103check="ok"
104
105#failed codes
106#sequence build status
107Fbuildsequ=1
108Fdoexcl=2
109#run process status
110Ftimecorr=3
111Ffillraw=4
112Fsinope=5
113Ffillsinope=6
114Fresetexcl=7
115#sequence process status
116Fwritesequfile=8
117Ffilesavail=9
118Fnoccfile=10
119Fnocacofile=11
120Fmerppcc=12
121Fmerppcaco=13
122Fcallisto=14
123Ffillcalib=15
124Ffillsignal=16
125Fstar=17
126Ffillstar=18
127#dataset process status
128Fwritedatasetfile=19
129Fstardone=20
130Fganymed=21
131Ffillganymed=22
132#again run process status
133FCompmux=26
134Fdowebplots=27
135#again mc run process status
136Fmccallisto=28
137Ffillmccalib=29
138Ffillmcsignal=30
139Fmcstar=31
140Ffillmcstar=32
141Fcorsikasimtel=33
142Fchimp=34
143Fchimpcp=35
144Fctastar=36
145Fctastarcp=37
146FctastereoA=38
147FctastereoB=39
148FctastereoC=40
149FctastereoD=41
150FctastereoE=42
151FctastereoF=43
152FctastereoG=44
153FctastereoH=45
154Fctastereocp=46
155
156# setup for jobmanager:
157# log files (can't be defined in script itself, as script can run longer
158# than one day
159jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
160jmscriptlog=$runlogpath/jobmanager`date +%F`.log
161
162# check if rc-files are available
163if ! ls $steps >/dev/null
164then
165 echo "Can't find steps.rc ($steps)"
166 finish
167fi
168if ! ls $sqlrc >/dev/null
169then
170 echo "Can't find sql.rc ($sqlrc)"
171 finish
172fi
173
174# resetting values
175pno=0
176totalpno=0
177running=0
178queued=0
179runningscript=0
180queuedscript=0
181stillinqueue=0
182
183
184# alias (we cannot check the beginning of the line due to
185# color codes in filldotraw.C)
186alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
187
188
189# in the following the functions, which are needed by several scripts, are
190# defined
191
192# function to check if a process is already locked
193# command line option can be used to execute something, e.g. 'continue'
194function checklock()
195{
196 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
197 then
198 printprocesslog "WARN lockfile $lockfile exists"
199 $@
200 exit
201 else
202 printprocesslog "INFO created lockfile $lockfile"
203 fi
204}
205
206function resetstatusvalues()
207{
208 statustime=NULL
209 starttime=NULL
210 returncode=NULL
211 programid=NULL
212 failedtime=NULL
213}
214
215function printstatusvalues()
216{
217 echo "the current values are:"
218 echo " statustime=$statustime"
219 echo " starttime=$starttime"
220 echo " returncode=$returncode"
221 echo " programid=$programid"
222 echo " failedtime=$failedtime"
223 echo "-- check: -$check-"
224 echo ""
225}
226
227# function evaluating the statusvalues
228function evalstatus()
229{
230 case $@ in
231 start) printprocesslog "INFO setstatus start"
232 starttime="Now()"
233 ;;
234 stop) case $check in
235 ok) printprocesslog "INFO setstatus stop - ok"
236 statustime="Now()"
237 ;;
238 no) printprocesslog "INFO setstatus stop - nothing new"
239 check="ok"
240 ;;
241 *) printprocesslog "INFO setstatus stop - failed"
242 starttime=noreset
243 if [ "$check" == "" ]
244 then
245 returncode=1
246 else
247 returncode=$check
248 fi
249 programid=$com
250 failedtime="Now()"
251 check="ok"
252 ;;
253 esac
254 ;;
255 *) printprocesslog "ERROR function evalstatus got wrong variable"
256 finish
257 ;;
258 esac
259}
260
261function getdbsetup()
262{
263 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
264 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
265 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
266 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
267# echo "setup: "
268# echo " db: "$db
269# echo " pw: "$pw
270# echo " us: "$us
271# echo " ho: "$ho
272}
273
274function getstepinfo()
275{
276 table=`grep "$column:" $steps | sed -e "s/[.]$column://" -e 's/#//' -e 's/ //g'`
277 coltab=`grep "$column:" $steps | sed -e 's/://' -e 's/#//' -e 's/ //g'`
278 needs=`grep "$coltab[.]Needs:" $steps | sed -e "s/$coltab[.]Needs://"`
279 influences=`grep "$coltab[.]Influences:" $steps | sed -e "s/$coltab[.]Influences://"`
280 prims=( `grep "$table[.]Primary:" $steps | sed -e "s/$table[.]Primary://"` )
281# echo " column $column - table $table - coltab $coltab"
282# echo " needs: $needs"
283# echo " influences: $influences"
284# echo " prims: ${prims[@]}"
285}
286
287# function to get todolist
288function gettodo()
289{
290 process=
291 printprocesslog "INFO getting todo..."
292 getdbsetup
293 getstepinfo
294 # get query
295 query=" SELECT "${prims[0]}
296 for (( i=1 ; i < ${#prims[@]} ; i++ ))
297 do
298 query=$query", ${prims[$i]}"
299 done
300 query=$query" FROM $table WHERE "
301 if ! echo $needs | grep '#' > /dev/null
302 then
303 for need in $needs
304 do
305 query=$query" NOT ISNULL($need) AND"
306 done
307 fi
308 query=$query" ISNULL($column) "
309 if [ "$2 " != " " ]
310 then
311 query=$query" AND fProductionHostKEY=$2 "
312 fi
313 query=$query" AND ISNULL(fStartTime) AND ISNULL(fFailedTime) AND ISNULL(fProgramId) AND ISNULL(fReturnCode) "
314 query=$query" ORDER BY fPriority desc "
315 if [ "$1 " != " " ]
316 then
317 query=$query" limit 0, $1 "
318 fi
319# echo " QUERY: "$query
320 printprocesslog "INFO gettodo QUERY: "$query
321 if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
322 then
323 printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
324 finish
325 fi
326
327 if [ "$process" = "" ]
328 then
329 printprocesslog "INFO => nothing to do"
330 finish
331 else
332 primaries=( $process )
333 num=`expr ${#primaries[@]} / ${#prims[@]} `
334 fi
335}
336
337
338# function to get the number of processes which still have to be done
339function getstatus()
340{
341 numproc=
342 getdbsetup
343 getstepinfo
344 # get query
345 query=" SELECT COUNT(*) FROM $table WHERE "
346 if ! echo $needs | grep '#' > /dev/null
347 then
348 for need in $needs
349 do
350 query=$query" NOT ISNULL($need) AND"
351 done
352 fi
353 query=$query" ISNULL($column) "
354 if [ "$1 " != " " ]
355 then
356 query=$query" AND fProductionHostKEY=$1 "
357 fi
358 query=$query" AND ISNULL(fStartTime) AND ISNULL(fFailedTime) AND ISNULL(fProgramId) AND ISNULL(fReturnCode) "
359 query=$query" GROUP BY $column "
360# echo "QUERY: "$query
361 printprocesslog "INFO getstatus QUERY: "$query
362 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
363 then
364 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
365 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
366 continue
367 fi
368}
369
370# function to set status of a process in the db
371function setstatus()
372{
373 if [ "$column" = "no" ]
374 then
375 return
376 fi
377 resetstatusvalues
378 evalstatus $@
379 getdbsetup
380 getstepinfo
381 # get query
382 reset=`grep "$coltab[.]Reset:" $steps | sed -e "s/$coltab[.]Reset://" -e 's/ //g'`
383 if [ "$reset" = "no" ]
384 then
385 printprocesslog "ERROR You cannot reset $column for ${primaries[$s+$s]}"
386 finish
387 fi
388 query=" update $table set $column=$statustime"
389 if ! echo $influences | grep '#' > /dev/null
390 then
391 for influence in $influences
392 do
393 query=$query", $influence=NULL"
394 done
395 fi
396 if ! [ "$starttime" = "noreset" ]
397 then
398 query=$query", fStartTime=$starttime"
399 fi
400 query=$query", fFailedTime=$failedtime, fProgramId=$programid, fReturnCode=$returncode "
401 query=$query" where "
402 if [ "$s" = "" ]
403 then
404 s=0
405 fi
406 query=$query" ${prims[0]}='${primaries[$s*${#prims[@]}]}'"
407 for (( i=1 ; i < ${#prims[@]} ; i++ ))
408 do
409 query=$query" AND ${prims[$i]}='${primaries[$s*${#prims[@]}+$i]}' "
410 done
411# echo " QUERY: "$query
412 printprocesslog "INFO setstatus QUERY: "$query
413 if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
414 then
415 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
416 finish
417 fi
418
419}
420
421# function to send a mysql query
422function sendquery()
423{
424 getdbsetup
425 printprocesslog "INFO sendquery QUERY: "$query
426 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
427 then
428 printprocesslog "ERROR could not query db (program: $program, function sendquery)"
429 return 1
430 fi
431 if [ "$val" = "NULL" ]
432 then
433 val=
434 fi
435 echo $val
436 return 0
437}
438
Note: See TracBrowser for help on using the repository browser.