source: trunk/DataCheck/Sourcefile.sh@ 13721

Last change on this file since 13721 was 13310, checked in by Daniela Dorner, 12 years ago
warn only if lockfile older than 5 minutes
  • Property svn:executable set to *
File size: 13.1 KB
Line 
1#!/bin/bash
2
3# to treat aliases in bash-script correctly
4shopt -s expand_aliases
5
6# check if script has been started with absolute path
7if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
8then
9 echo "Please start your script with an absolute path."
10 exit
11fi
12
13if [ "$AUTOMATIONSETUP" = "" ]
14then
15 echo "Please set the environment variable \$AUTOMATIONSETUP."
16 exit
17fi
18
19if [ "$SOURCEFILEPATH" = "" ]
20then
21 export SOURCEFILEPATH=`dirname $0`
22fi
23if [ "$SCRIPTNAME" = "" ]
24then
25 SCRIPTNAME=`basename $0`
26fi
27
28source $SOURCEFILEPATH/../Setup/setup.$AUTOMATIONSETUP
29
30if [ "$mars" = "" ]
31then
32 echo "Please set the path for MARS."
33 exit
34fi
35
36datetime=`date +%F-%H-%M-%S`
37
38
39# function to make sure that a directory is made
40function makedir()
41{
42 if [ ! -d $@ ]
43 then
44 if [ "$processlog" = "" ]
45 then
46 mkdir -p $@
47 else
48 mkdir -pv $@
49 fi
50 if [ ! -d $@ ]
51 then
52 if ! [ "$processlog" = "" ]
53 then
54 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
55 else
56 echo "could not make dir "$@
57 fi
58 if ls $lockfile >/dev/null 2>&1
59 then
60 rm -v $lockfile
61 fi
62 exit
63 fi
64 fi
65}
66
67# logging paths for runlogs and processlog
68runlogpath=$logpath/run/`date +%Y/%m/%d`
69processlogpath=$logpath/processlog
70makedir $runlogpath
71makedir $processlogpath
72processlog=$processlogpath/process`date +%F`.log
73
74makedir $lockpath
75
76
77# function to provide proper logging in a single logfile ($processlog)
78function printprocesslog
79{
80 makedir $processlogpath
81 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
82}
83
84# function to exit a script properly
85function finish()
86{
87 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
88 then
89 printprocesslog "DEBUG " `rm -v $lockfile`
90 fi
91 printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
92 exit
93}
94
95
96# set checkvalue to ok at the beginning of the scripts
97check="ok"
98
99# setup for jobmanager:
100# log files (can't be defined in script itself, as script can run longer
101# than one day
102jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
103jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
104
105# check if rc-files are available
106if ! ls $steps >/dev/null
107then
108 echo "Can't find steps.rc ($steps)"
109 finish
110fi
111if ! ls $sqlrc >/dev/null
112then
113 echo "Can't find sql.rc ($sqlrc)"
114 finish
115fi
116
117# resetting values for jobmanager
118pno=0
119totalpno=0
120running=0
121queued=0
122runningscript=0
123queuedscript=0
124stillinqueue=0
125
126
127# alias (we cannot check the beginning of the line due to
128# color codes in filldotraw.C)
129alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
130
131
132# in the following the functions, which are needed by several scripts, are
133# defined
134
135# function to check if a process is already locked
136# command line option can be used to execute something, e.g. 'continue'
137function checklock()
138{
139 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
140 then
141 if find $lockfile -amin -5
142 then
143 printprocesslog "INFO lockfile $lockfile exists"
144 else
145 printprocesslog "WARN lockfile $lockfile exists"
146 fi
147 $@
148 exit
149 else
150 printprocesslog "DEBUG created lockfile $lockfile"
151 fi
152}
153
154# print the current status values
155function printstatusvalues()
156{
157 echo "the current values are:"
158 echo " starttime=$starttime"
159 echo " stoptime=$stoptime"
160 echo " availtime=$availtime"
161 echo " returncode=$returncode"
162 echo "-- check: -$check-"
163 echo ""
164}
165
166# get the db-setup from the sql.rc
167function getdbsetup()
168{
169 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
170 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
171 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
172 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
173# echo "setup: "
174# echo " db: "$db
175# echo " pw: "$pw
176# echo " us: "$us
177# echo " ho: "$ho
178}
179
180# function to send a mysql query
181function sendquery()
182{
183 getdbsetup
184 printprocesslog "DEBUG sendquery QUERY: "$query
185 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
186 then
187 printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us
188 #printprocesslog "ERROR could not query db (program: $program, function sendquery)"
189 #return 1 #why???
190 finish
191 fi
192 if [ "$val" = "NULL" ]
193 then
194 val=
195 fi
196 echo $val
197 return 0
198}
199
200# function to get information from the setupfile $steps
201function getfromsetup()
202{
203 grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
204}
205
206# function to get the needed information from the dependencies-file steps.rc
207function getstepinfo()
208{
209 getdbsetup
210 needs=( `getfromsetup $step "Needs"` )
211 noderestricted=`getfromsetup $step "NodeRestricted"`
212 prims=( `getfromsetup $step "Primaries"` )
213 maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
214# echo " maintable: "$maintable
215# echo " needs: "${needs[@]}
216# echo " noderestricted: "$noderestricted
217# echo " prims: "${prims[@]}
218}
219
220# function to get the joins needed for the get/set status queries
221function getalljoins()
222{
223 # add table
224 query=$query" "$maintable"Status"
225 # add special join
226 query=$query" "`getfromsetup $maintable "SpecialJoin"`
227 # add join for step unless step is the same as maintable
228 if ! [ "$step" = "$maintable" ]
229 then
230 query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
231 fi
232 # add joins for influences or needs
233 for otherstep in ${othersteps[@]}
234 do
235 if ! [ "$otherstep" = "$maintable" ]
236 then
237 query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
238 fi
239 done
240}
241
242# function to create the middle part of a query
243# which is identical for the functions getstatus() and gettodo()
244function getstatusquery()
245{
246 # add from which table the information is queried
247 query=$query" FROM "
248 othersteps=${needs[@]}
249 getalljoins
250 # add condition
251 query=$query" WHERE "
252 # add condition for step, i.e. step is not yet done
253 query=$query" ISNULL("$step"Status.fStartTime) "
254 query=$query" AND ISNULL("$step"Status.fStopTime) "
255 query=$query" AND ISNULL("$step"Status.fAvailable) "
256 query=$query" AND ISNULL("$step"Status.fReturnCode) "
257 # add requirement for production host in case it is needed
258 if [ "$1 " != " " ]
259 then
260 query=$query" AND fProductionHostKEY=$2 "
261 fi
262 if ! echo $query | grep UPDATE >/dev/null 2>&1
263 then
264 query=$query" GROUP BY "${prims[@]}
265 fi
266 # add condition for needs, i.e. that step is done
267 for (( k=0 ; k < ${#needs[@]} ; k++ ))
268 do
269 if [ $k -eq 0 ]
270 then
271 query=$query" HAVING "
272 else
273 query=$query" AND "
274 fi
275 query=$query" COUNT(*)=COUNT(IF("
276 query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
277 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
278 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
279 query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
280 query=$query" , 1, NULL)) "
281 done
282}
283
284# function to get todolist
285# returns the next or the list of next steps
286function gettodo()
287{
288 # reset the variable for the number of the next step
289 process=
290 printprocesslog "DEBUG getting todo for step $step..."
291 getstepinfo
292 # get query
293 query=" SELECT "${prims[@]}
294 getstatusquery $2
295 # order by priority to the the number of the next step to be done
296 query=$query" ORDER BY "$step"Status.fPriority desc "
297 # add limitation in case only one or a limited number of
298 # processes should be executed
299 if [ "$1 " != " " ]
300 then
301 query=$query" limit 0, $1 "
302 fi
303 # print query
304 printprocesslog "DEBUG gettodo for step $step QUERY: "$query
305 # execute query
306 process=`sendquery`
307 #if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
308 #then
309 # printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
310 # finish
311 #fi
312 # get numbers of next step from mysql result
313 if [ "$process" = "" ]
314 then
315 printprocesslog "DEBUG => nothing to do"
316 finish
317 else
318 primaries=( $process )
319 num=`expr ${#primaries[@]} / ${#prims[@]} `
320 fi
321}
322
323# function to get the number of processes which still have to be done
324function getstatus()
325{
326 # reset the variable for the number of steps to be done
327 numproc=0
328 getstepinfo
329 # get query
330 query=" SELECT "${prims[@]}
331 getstatusquery $1
332 # print query
333 printprocesslog "DEBUG getstatus for step $step QUERY: "$query
334 # execute query
335 #numproc=`sendquery `#cannot be done with sendquery, because of row counting
336 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query " | wc -l`
337 then
338 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
339 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
340 continue
341 fi
342}
343
344# function to set status of a process in the db
345function setstatus()
346{
347 # remark:
348 # this function does not include the 'Default' flag
349 # for resetting steps
350
351 # for dowebplots (there are steps which have no entry in the DB)
352 if [ "$step" = "no" ]
353 then
354 return
355 fi
356
357 # reset status values
358 starttime=NULL
359 stoptime=NULL
360 availtime=NULL
361 returncode=NULL
362 # evaluate the status values
363 case $@ in
364 start) printprocesslog "DEBUG setstatus start"
365 starttime="Now()"
366 ;;
367 stop) case $check in
368 ok) printprocesslog "DEBUB setstatus stop - ok"
369 starttime=noreset
370 stoptime="Now()"
371 if [ "$processingsite" = "$storagesite" ]
372 then
373 availtime="Now()"
374 fi
375 ;;
376 no) printprocesslog "DEBUG setstatus stop - nothing new"
377 check="ok"
378 ;;
379 *) printprocesslog "DEBUG setstatus stop - failed"
380 starttime=noreset
381 stoptime="Now()"
382 if [ "$processingsite" = "$storagesite" ]
383 then
384 availtime="Now()"
385 fi
386 if [ "$check" == "" ]
387 then
388 returncode=1
389 else
390 returncode=$check
391 fi
392 check="ok"
393 ;;
394 esac
395 ;;
396 *) printprocesslog "ERROR function setstatus got wrong variable"
397 finish
398 ;;
399 esac
400
401 # get
402 getstepinfo
403
404 # get the influences from the steps.rc by evaluating the needs of all steps
405 othersteps=`grep $step $steps | grep -v '#' | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
406
407 # get query
408 query=" UPDATE "
409 getalljoins
410 # set the status values according to the new status of the step
411 query=$query" SET "
412 if ! [ "$starttime" = "noreset" ]
413 then
414 query=$query" "$step"Status.fStartTime=$starttime, "
415 fi
416 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
417 query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
418 # set also the status values of the influenced steps
419 for otherstep in $othersteps
420 do
421 query=$query", "$otherstep"Status.fStartTime=NULL "
422 query=$query", "$otherstep"Status.fStopTime=NULL "
423 query=$query", "$otherstep"Status.fAvailable=NULL "
424 query=$query", "$otherstep"Status.fReturnCode=NULL "
425 query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
426 done
427 # give the condition for which step the status values have to be set
428 query=$query" WHERE "
429 if [ "$s" = "" ]
430 then
431 s=0
432 fi
433 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
434 for (( j=1 ; j < ${#prims[@]} ; j++ ))
435 do
436 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
437 done
438 # add additional query to allow for locking in db
439 if [ "$1" = "start" ]
440 then
441 query=$query" AND ISNULL("$step"Status.fStartTime) "
442 fi
443 # add row count to know how many rows have been changed
444 query=$query"; SELECT ROW_COUNT();"
445 # print query
446 printprocesslog "DEBUG setstatus for step $step QUERY: "$query
447 # execute query
448 numchanged=`sendquery`
449 #if ! numchanged=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
450 #then
451 # printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
452 # finish
453 #fi
454 if [ $numchanged -gt 0 ]
455 then
456 printprocesslog "INFO successful set of status in DB."
457 else
458 # should not happen
459 printprocesslog "ERROR status in DB was already set by another process "
460 fi
461}
462
463
Note: See TracBrowser for help on using the repository browser.