source: trunk/DataCheck/Sourcefile.sh@ 13305

Last change on this file since 13305 was 13302, checked in by Daniela Dorner, 14 years ago
removed things which belong into setup files
  • Property svn:executable set to *
File size: 13.0 KB
Line 
1#!/bin/bash
2
3# to treat aliases in bash-script correctly
4shopt -s expand_aliases
5
6# check if script has been started with absolute path
7if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
8then
9 echo "Please start your script with an absolute path."
10 exit
11fi
12
13if [ "$AUTOMATIONSETUP" = "" ]
14then
15 echo "Please set the environment variable \$AUTOMATIONSETUP."
16 exit
17fi
18
19if [ "$SOURCEFILEPATH" = "" ]
20then
21 export SOURCEFILEPATH=`dirname $0`
22fi
23if [ "$SCRIPTNAME" = "" ]
24then
25 SCRIPTNAME=`basename $0`
26fi
27
28source $SOURCEFILEPATH/../Setup/setup.$AUTOMATIONSETUP
29
30if [ "$mars" = "" ]
31then
32 echo "Please set the path for MARS."
33 exit
34fi
35
36datetime=`date +%F-%H-%M-%S`
37
38
39# function to make sure that a directory is made
40function makedir()
41{
42 if [ ! -d $@ ]
43 then
44 if [ "$processlog" = "" ]
45 then
46 mkdir -p $@
47 else
48 mkdir -pv $@
49 fi
50 if [ ! -d $@ ]
51 then
52 if ! [ "$processlog" = "" ]
53 then
54 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
55 else
56 echo "could not make dir "$@
57 fi
58 if ls $lockfile >/dev/null 2>&1
59 then
60 rm -v $lockfile
61 fi
62 exit
63 fi
64 fi
65}
66
67# logging paths for runlogs and processlog
68runlogpath=$logpath/run/`date +%Y/%m/%d`
69processlogpath=$logpath/processlog
70makedir $runlogpath
71makedir $processlogpath
72processlog=$processlogpath/process`date +%F`.log
73
74makedir $lockpath
75
76
77# function to provide proper logging in a single logfile ($processlog)
78function printprocesslog
79{
80 makedir $processlogpath
81 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
82}
83
84# function to exit a script properly
85function finish()
86{
87 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
88 then
89 printprocesslog "DEBUG " `rm -v $lockfile`
90 fi
91 printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
92 exit
93}
94
95
96# set checkvalue to ok at the beginning of the scripts
97check="ok"
98
99# setup for jobmanager:
100# log files (can't be defined in script itself, as script can run longer
101# than one day
102jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
103jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
104
105# check if rc-files are available
106if ! ls $steps >/dev/null
107then
108 echo "Can't find steps.rc ($steps)"
109 finish
110fi
111if ! ls $sqlrc >/dev/null
112then
113 echo "Can't find sql.rc ($sqlrc)"
114 finish
115fi
116
117# resetting values for jobmanager
118pno=0
119totalpno=0
120running=0
121queued=0
122runningscript=0
123queuedscript=0
124stillinqueue=0
125
126
127# alias (we cannot check the beginning of the line due to
128# color codes in filldotraw.C)
129alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
130
131
132# in the following the functions, which are needed by several scripts, are
133# defined
134
135# function to check if a process is already locked
136# command line option can be used to execute something, e.g. 'continue'
137function checklock()
138{
139 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
140 then
141 printprocesslog "WARN lockfile $lockfile exists"
142 $@
143 exit
144 else
145 printprocesslog "DEBUG created lockfile $lockfile"
146 fi
147}
148
149# print the current status values
150function printstatusvalues()
151{
152 echo "the current values are:"
153 echo " starttime=$starttime"
154 echo " stoptime=$stoptime"
155 echo " availtime=$availtime"
156 echo " returncode=$returncode"
157 echo "-- check: -$check-"
158 echo ""
159}
160
161# get the db-setup from the sql.rc
162function getdbsetup()
163{
164 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
165 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
166 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
167 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
168# echo "setup: "
169# echo " db: "$db
170# echo " pw: "$pw
171# echo " us: "$us
172# echo " ho: "$ho
173}
174
175# function to send a mysql query
176function sendquery()
177{
178 getdbsetup
179 printprocesslog "DEBUG sendquery QUERY: "$query
180 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
181 then
182 printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us
183 #printprocesslog "ERROR could not query db (program: $program, function sendquery)"
184 #return 1 #why???
185 finish
186 fi
187 if [ "$val" = "NULL" ]
188 then
189 val=
190 fi
191 echo $val
192 return 0
193}
194
195# function to get information from the setupfile $steps
196function getfromsetup()
197{
198 grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
199}
200
201# function to get the needed information from the dependencies-file steps.rc
202function getstepinfo()
203{
204 getdbsetup
205 needs=( `getfromsetup $step "Needs"` )
206 noderestricted=`getfromsetup $step "NodeRestricted"`
207 prims=( `getfromsetup $step "Primaries"` )
208 maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
209# echo " maintable: "$maintable
210# echo " needs: "${needs[@]}
211# echo " noderestricted: "$noderestricted
212# echo " prims: "${prims[@]}
213}
214
215# function to get the joins needed for the get/set status queries
216function getalljoins()
217{
218 # add table
219 query=$query" "$maintable"Status"
220 # add special join
221 query=$query" "`getfromsetup $maintable "SpecialJoin"`
222 # add join for step unless step is the same as maintable
223 if ! [ "$step" = "$maintable" ]
224 then
225 query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
226 fi
227 # add joins for influences or needs
228 for otherstep in ${othersteps[@]}
229 do
230 if ! [ "$otherstep" = "$maintable" ]
231 then
232 query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
233 fi
234 done
235}
236
237# function to create the middle part of a query
238# which is identical for the functions getstatus() and gettodo()
239function getstatusquery()
240{
241 # add from which table the information is queried
242 query=$query" FROM "
243 othersteps=${needs[@]}
244 getalljoins
245 # add condition
246 query=$query" WHERE "
247 # add condition for step, i.e. step is not yet done
248 query=$query" ISNULL("$step"Status.fStartTime) "
249 query=$query" AND ISNULL("$step"Status.fStopTime) "
250 query=$query" AND ISNULL("$step"Status.fAvailable) "
251 query=$query" AND ISNULL("$step"Status.fReturnCode) "
252 # add requirement for production host in case it is needed
253 if [ "$1 " != " " ]
254 then
255 query=$query" AND fProductionHostKEY=$2 "
256 fi
257 if ! echo $query | grep UPDATE >/dev/null 2>&1
258 then
259 query=$query" GROUP BY "${prims[@]}
260 fi
261 # add condition for needs, i.e. that step is done
262 for (( k=0 ; k < ${#needs[@]} ; k++ ))
263 do
264 if [ $k -eq 0 ]
265 then
266 query=$query" HAVING "
267 else
268 query=$query" AND "
269 fi
270 query=$query" COUNT(*)=COUNT(IF("
271 query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
272 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
273 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
274 query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
275 query=$query" , 1, NULL)) "
276 done
277}
278
279# function to get todolist
280# returns the next or the list of next steps
281function gettodo()
282{
283 # reset the variable for the number of the next step
284 process=
285 printprocesslog "DEBUG getting todo for step $step..."
286 getstepinfo
287 # get query
288 query=" SELECT "${prims[@]}
289 getstatusquery $2
290 # order by priority to the the number of the next step to be done
291 query=$query" ORDER BY "$step"Status.fPriority desc "
292 # add limitation in case only one or a limited number of
293 # processes should be executed
294 if [ "$1 " != " " ]
295 then
296 query=$query" limit 0, $1 "
297 fi
298 # print query
299 printprocesslog "DEBUG gettodo for step $step QUERY: "$query
300 # execute query
301 process=`sendquery`
302 #if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
303 #then
304 # printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
305 # finish
306 #fi
307 # get numbers of next step from mysql result
308 if [ "$process" = "" ]
309 then
310 printprocesslog "DEBUG => nothing to do"
311 finish
312 else
313 primaries=( $process )
314 num=`expr ${#primaries[@]} / ${#prims[@]} `
315 fi
316}
317
318# function to get the number of processes which still have to be done
319function getstatus()
320{
321 # reset the variable for the number of steps to be done
322 numproc=0
323 getstepinfo
324 # get query
325 query=" SELECT "${prims[@]}
326 getstatusquery $1
327 # print query
328 printprocesslog "DEBUG getstatus for step $step QUERY: "$query
329 # execute query
330 #numproc=`sendquery `#cannot be done with sendquery, because of row counting
331 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query " | wc -l`
332 then
333 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
334 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
335 continue
336 fi
337}
338
339# function to set status of a process in the db
340function setstatus()
341{
342 # remark:
343 # this function does not include the 'Default' flag
344 # for resetting steps
345
346 # for dowebplots (there are steps which have no entry in the DB)
347 if [ "$step" = "no" ]
348 then
349 return
350 fi
351
352 # reset status values
353 starttime=NULL
354 stoptime=NULL
355 availtime=NULL
356 returncode=NULL
357 # evaluate the status values
358 case $@ in
359 start) printprocesslog "DEBUG setstatus start"
360 starttime="Now()"
361 ;;
362 stop) case $check in
363 ok) printprocesslog "DEBUB setstatus stop - ok"
364 starttime=noreset
365 stoptime="Now()"
366 if [ "$processingsite" = "$storagesite" ]
367 then
368 availtime="Now()"
369 fi
370 ;;
371 no) printprocesslog "DEBUG setstatus stop - nothing new"
372 check="ok"
373 ;;
374 *) printprocesslog "DEBUG setstatus stop - failed"
375 starttime=noreset
376 stoptime="Now()"
377 if [ "$processingsite" = "$storagesite" ]
378 then
379 availtime="Now()"
380 fi
381 if [ "$check" == "" ]
382 then
383 returncode=1
384 else
385 returncode=$check
386 fi
387 check="ok"
388 ;;
389 esac
390 ;;
391 *) printprocesslog "ERROR function setstatus got wrong variable"
392 finish
393 ;;
394 esac
395
396 # get
397 getstepinfo
398
399 # get the influences from the steps.rc by evaluating the needs of all steps
400 othersteps=`grep $step $steps | grep -v '#' | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
401
402 # get query
403 query=" UPDATE "
404 getalljoins
405 # set the status values according to the new status of the step
406 query=$query" SET "
407 if ! [ "$starttime" = "noreset" ]
408 then
409 query=$query" "$step"Status.fStartTime=$starttime, "
410 fi
411 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
412 query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
413 # set also the status values of the influenced steps
414 for otherstep in $othersteps
415 do
416 query=$query", "$otherstep"Status.fStartTime=NULL "
417 query=$query", "$otherstep"Status.fStopTime=NULL "
418 query=$query", "$otherstep"Status.fAvailable=NULL "
419 query=$query", "$otherstep"Status.fReturnCode=NULL "
420 query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
421 done
422 # give the condition for which step the status values have to be set
423 query=$query" WHERE "
424 if [ "$s" = "" ]
425 then
426 s=0
427 fi
428 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
429 for (( j=1 ; j < ${#prims[@]} ; j++ ))
430 do
431 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
432 done
433 # add additional query to allow for locking in db
434 if [ "$1" = "start" ]
435 then
436 query=$query" AND ISNULL("$step"Status.fStartTime) "
437 fi
438 # add row count to know how many rows have been changed
439 query=$query"; SELECT ROW_COUNT();"
440 # print query
441 printprocesslog "DEBUG setstatus for step $step QUERY: "$query
442 # execute query
443 numchanged=`sendquery`
444 #if ! numchanged=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
445 #then
446 # printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
447 # finish
448 #fi
449 if [ $numchanged -gt 0 ]
450 then
451 printprocesslog "INFO successful set of status in DB."
452 else
453 # should not happen
454 printprocesslog "ERROR status in DB was already set by another process "
455 fi
456}
457
458
Note: See TracBrowser for help on using the repository browser.