source: trunk/DataCheck/Sourcefile.sh@ 16483

Last change on this file since 16483 was 15600, checked in by Daniela Dorner, 11 years ago
updated debugging output and logging for case that job is already done by other script
  • Property svn:executable set to *
File size: 13.5 KB
Line 
1#!/bin/bash
2
3# to treat aliases in bash-script correctly
4shopt -s expand_aliases
5
6# check if script has been started with absolute path
7if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
8then
9 echo "Please start your script with an absolute path."
10 exit
11fi
12
13if [ "$AUTOMATIONSETUP" = "" ]
14then
15 echo "Please set the environment variable \$AUTOMATIONSETUP."
16 exit
17fi
18
19if [ "$SOURCEFILEPATH" = "" ]
20then
21 export SOURCEFILEPATH=`dirname $0`
22fi
23if [ "$SCRIPTNAME" = "" ]
24then
25 SCRIPTNAME=`basename $0`
26fi
27
28source $SOURCEFILEPATH/../Setup/setup.$AUTOMATIONSETUP
29
30if [ "$mars" = "" ]
31then
32 echo "Please set the path for MARS."
33 exit
34fi
35
36datetime=`date +%F-%H-%M-%S`
37
38
39# function to make sure that a directory is made
40function makedir()
41{
42 if [ ! -d $@ ]
43 then
44 if [ "$processlog" = "" ]
45 then
46 mkdir -p $@
47 else
48 mkdir -pv $@
49 fi
50 if [ ! -d $@ ]
51 then
52 if ! [ "$processlog" = "" ]
53 then
54 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
55 else
56 echo "could not make dir "$@
57 fi
58 if ls $lockfile >/dev/null 2>&1
59 then
60 rm -v $lockfile
61 fi
62 exit
63 fi
64 fi
65}
66
67# logging paths for runlogs and processlog
68runlogpath=$logpath/run/`date +%Y/%m/%d`
69processlogpath=$logpath/processlog
70makedir $runlogpath
71makedir $processlogpath
72processlog=$processlogpath/process`date +%F`.log
73
74makedir $lockpath
75
76
77# function to provide proper logging in a single logfile ($processlog)
78function printprocesslog
79{
80 makedir $processlogpath
81 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
82}
83
84# function to exit a script properly
85function finish()
86{
87 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
88 then
89 printprocesslog "DEBUG " `rm -v $lockfile`
90 fi
91 printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
92 exit
93}
94
95
96# set checkvalue to ok at the beginning of the scripts
97check="ok"
98
99# setup for jobmanager:
100# log files (can't be defined in script itself, as script can run longer
101# than one day
102jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
103jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
104
105# check if rc-files are available
106if ! ls $steps >/dev/null
107then
108 echo "Can't find steps.rc ($steps)"
109 finish
110fi
111if ! ls $sqlrc >/dev/null
112then
113 echo "Can't find sql.rc ($sqlrc)"
114 finish
115fi
116
117# resetting values for jobmanager
118pno=0
119totalpno=0
120running=0
121queued=0
122runningscript=0
123queuedscript=0
124stillinqueue=0
125
126
127# alias (we cannot check the beginning of the line due to
128# color codes in filldotraw.C)
129alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
130
131
132# in the following the functions, which are needed by several scripts, are
133# defined
134
135# function to check if a process is already locked
136# command line option can be used to execute something, e.g. 'continue'
137function checklock()
138{
139 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
140 then
141 if find $lockfile -amin -5
142 then
143 printprocesslog "INFO lockfile $lockfile exists"
144 else
145 printprocesslog "WARN lockfile $lockfile exists"
146 fi
147 $@
148 exit
149 else
150 printprocesslog "DEBUG created lockfile $lockfile"
151 fi
152}
153
154# print the current status values
155function printstatusvalues()
156{
157 echo "the current values are:"
158 echo " starttime=$starttime"
159 echo " stoptime=$stoptime"
160 echo " availtime=$availtime"
161 echo " returncode=$returncode"
162 echo "-- check: -$check-"
163 echo ""
164}
165
166# get the db-setup from the sql.rc
167function getdbsetup()
168{
169 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
170 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
171 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
172 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
173# echo "setup: "
174# echo " db: "$db
175# echo " pw: "$pw
176# echo " us: "$us
177# echo " ho: "$ho
178}
179
180# function to send a mysql query
181function sendquery()
182{
183 getdbsetup
184 printprocesslog "DEBUG sendquery QUERY: "$query
185 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
186 then
187 printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us
188 #printprocesslog "ERROR could not query db (program: $program, function sendquery)"
189 #return 1 #why???
190 finish
191 fi
192 if [ "$val" = "NULL" ]
193 then
194 val=
195 fi
196 echo $val
197 return 0
198}
199
200# function to get information from the setupfile $steps
201function getfromsetup()
202{
203 grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
204}
205
206# function to get the needed information from the dependencies-file steps.rc
207function getstepinfo()
208{
209 getdbsetup
210 needs=( `getfromsetup $step "Needs"` )
211 noderestricted=`getfromsetup $step "NodeRestricted"`
212 prims=( `getfromsetup $step "Primaries"` )
213 maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
214 sort=`getfromsetup $step "SortDirection" | sed -e "s/\ //g"`
215# echo " maintable: "$maintable
216# echo " needs: "${needs[@]}
217# echo " noderestricted: "$noderestricted
218# echo " prims: "${prims[@]}
219}
220
221# function to get the joins needed for the get/set status queries
222function getalljoins()
223{
224 # add table
225 query=$query" "$maintable"Status"
226 # add special join
227 query=$query" "`getfromsetup $maintable "SpecialJoin"`
228 # add join for step unless step is the same as maintable
229 if ! [ "$step" = "$maintable" ]
230 then
231 query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
232 fi
233 # add joins for influences or needs
234 for otherstep in ${othersteps[@]}
235 do
236 if ! [ "$otherstep" = "$maintable" ]
237 then
238 query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
239 fi
240 done
241}
242
243# function to create the middle part of a query
244# which is identical for the functions getstatus() and gettodo()
245function getstatusquery()
246{
247 # add from which table the information is queried
248 query=$query" FROM "
249 othersteps=${needs[@]}
250 getalljoins
251 # add condition
252 query=$query" WHERE "
253 # add condition for step, i.e. step is not yet done
254 query=$query" ISNULL("$step"Status.fStartTime) "
255 query=$query" AND ISNULL("$step"Status.fStopTime) "
256 query=$query" AND ISNULL("$step"Status.fAvailable) "
257 query=$query" AND ISNULL("$step"Status.fReturnCode) "
258 # add requirement for production host in case it is needed
259 if [ "$1 " != " " ]
260 then
261 query=$query" AND fProductionHostKEY=$2 "
262 fi
263 if ! echo $query | grep UPDATE >/dev/null 2>&1
264 then
265 query=$query" GROUP BY "${prims[@]}
266 fi
267 # add condition for needs, i.e. that step is done
268 for (( k=0 ; k < ${#needs[@]} ; k++ ))
269 do
270 if [ $k -eq 0 ]
271 then
272 query=$query" HAVING "
273 else
274 query=$query" AND "
275 fi
276 query=$query" COUNT(*)=COUNT(IF("
277 query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
278 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
279 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
280 query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
281 query=$query" , 1, NULL)) "
282 done
283}
284
285# function to get todolist
286# returns the next or the list of next steps
287function gettodo()
288{
289 # reset the variable for the number of the next step
290 process=
291 printprocesslog "DEBUG getting todo for step $step..."
292 getstepinfo
293 # get query
294 query=" SELECT "${prims[@]}
295 getstatusquery $2
296 # order by priority to the the number of the next step to be done
297 query=$query" ORDER BY "$step"Status.fPriority "
298 if [ "$sort" = "" ]
299 then
300 query=$query" DESC "
301 else
302 query=$query" "$sort
303 fi
304 # add limitation in case only one or a limited number of
305 # processes should be executed
306 if [ "$1 " != " " ]
307 then
308 query=$query" limit 0, $1 "
309 fi
310 # print query
311 printprocesslog "DEBUG gettodo for step $step QUERY: "$query
312 # execute query
313 process=`sendquery`
314 #if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
315 #then
316 # printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
317 # finish
318 #fi
319 # get numbers of next step from mysql result
320 if [ "$process" = "" ]
321 then
322 printprocesslog "DEBUG => nothing to do"
323 finish
324 else
325 primaries=( $process )
326 num=`expr ${#primaries[@]} / ${#prims[@]} `
327 fi
328}
329
330# function to get the number of processes which still have to be done
331function getstatus()
332{
333 # reset the variable for the number of steps to be done
334 numproc=0
335 getstepinfo
336 # get query
337 query=" SELECT "${prims[@]}
338 getstatusquery $1
339 # print query
340 printprocesslog "DEBUG getstatus for step $step QUERY: "$query
341 # execute query
342 #numproc=`sendquery `#cannot be done with sendquery, because of row counting
343 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query " | wc -l`
344 then
345 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
346 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
347 continue
348 fi
349}
350
351# function to set status of a process in the db
352function setstatus()
353{
354 # remark:
355 # this function does not include the 'Default' flag
356 # for resetting steps
357
358 # for dowebplots (there are steps which have no entry in the DB)
359 if [ "$step" = "no" ]
360 then
361 return
362 fi
363
364 # reset status values
365 starttime=NULL
366 stoptime=NULL
367 availtime=NULL
368 returncode=NULL
369 # evaluate the status values
370 case $@ in
371 start) printprocesslog "DEBUG setstatus start"
372 starttime="Now()"
373 ;;
374 stop) case $check in
375 ok) printprocesslog "DEBUB setstatus stop - ok"
376 starttime=noreset
377 stoptime="Now()"
378 if [ "$processingsite" = "$storagesite" ]
379 then
380 availtime="Now()"
381 fi
382 ;;
383 no) printprocesslog "DEBUG setstatus stop - nothing new"
384 check="ok"
385 ;;
386 *) printprocesslog "DEBUG setstatus stop - failed"
387 starttime=noreset
388 stoptime="Now()"
389 if [ "$processingsite" = "$storagesite" ]
390 then
391 availtime="Now()"
392 fi
393 if [ "$check" == "" ]
394 then
395 returncode=1
396 else
397 returncode=$check
398 fi
399 check="ok"
400 ;;
401 esac
402 ;;
403 *) printprocesslog "ERROR function setstatus got wrong variable"
404 finish
405 ;;
406 esac
407
408 # get
409 getstepinfo
410
411 # get the influences from the steps.rc by evaluating the needs of all steps
412 othersteps=`grep $step $steps | grep -v '#' | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
413
414 # get query
415 query=" UPDATE "
416 getalljoins
417 # set the status values according to the new status of the step
418 query=$query" SET "
419 if ! [ "$starttime" = "noreset" ]
420 then
421 query=$query" "$step"Status.fStartTime=$starttime, "
422 fi
423 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
424 query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
425 # set also the status values of the influenced steps
426 for otherstep in $othersteps
427 do
428 query=$query", "$otherstep"Status.fStartTime=NULL "
429 query=$query", "$otherstep"Status.fStopTime=NULL "
430 query=$query", "$otherstep"Status.fAvailable=NULL "
431 query=$query", "$otherstep"Status.fReturnCode=NULL "
432 query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
433 done
434 # give the condition for which step the status values have to be set
435 query=$query" WHERE "
436 if [ "$s" = "" ]
437 then
438 s=0
439 fi
440 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
441 for (( j=1 ; j < ${#prims[@]} ; j++ ))
442 do
443 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
444 done
445 # add additional query to allow for locking in db
446 if [ "$1" = "start" ]
447 then
448 query=$query" AND ISNULL("$step"Status.fStartTime) "
449 fi
450 # add row count to know how many rows have been changed
451 query=$query"; SELECT ROW_COUNT();"
452 # print query
453 printprocesslog "DEBUG setstatus for step $step QUERY: "$query
454 #echo "DEBUG setstatus for step $step QUERY: "$query
455 # execute query
456 numchanged=`sendquery`
457 #echo "numchanged: "$numchanged
458 #if ! numchanged=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
459 #then
460 # printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
461 # finish
462 #fi
463 if [ $numchanged -gt 0 ]
464 then
465 printprocesslog "INFO successful set of status in DB."
466 #echo "INFO successful set of status in DB."
467 else
468 # action may be taken in script using $numchanged
469 printprocesslog "ERROR status in DB was already set by another process "
470 #echo "ERROR status in DB was already set by another process "
471 fi
472}
473
474
Note: See TracBrowser for help on using the repository browser.