source: trunk/DataCheck/Sourcefile.sh@ 18099

Last change on this file since 18099 was 18065, checked in by Daniela Dorner, 10 years ago
improved logging
  • Property svn:executable set to *
File size: 14.7 KB
Line 
1#!/bin/bash
2
3# to treat aliases in bash-script correctly
4shopt -s expand_aliases
5
6# check if script has been started with absolute path
7if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
8then
9 echo "Please start your script with an absolute path."
10 exit
11fi
12
13if [ "$AUTOMATIONSETUP" = "" ]
14then
15 echo "Please set the environment variable \$AUTOMATIONSETUP."
16 exit
17fi
18
19if [ "$SOURCEFILEPATH" = "" ]
20then
21 export SOURCEFILEPATH=`dirname $0`
22fi
23if [ "$SCRIPTNAME" = "" ]
24then
25 SCRIPTNAME=`basename $0`
26fi
27
28source $SOURCEFILEPATH/../Setup/setup.$AUTOMATIONSETUP
29
30if [ "$mars" = "" ]
31then
32 echo "Please set the path for MARS."
33 exit
34fi
35
36datetime=`date +%F-%H-%M-%S`
37
38
39# function to make sure that a directory is made
40function makedir()
41{
42 if [ ! -d $@ ]
43 then
44 if [ "$processlog" = "" ] || [ "$logfile" = "" ]
45 then
46 mkdir -p $@
47 else
48 mkdir -pv $@
49 fi
50 if [ ! -d $@ ]
51 then
52 if ! [ "$processlog" = "" ]
53 then
54 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] ERROR could not make dir "$@ >> $processlog
55 else
56 echo "could not make dir "$@
57 fi
58 if ls $lockfile >/dev/null 2>&1
59 then
60 rm -v $lockfile
61 fi
62 exit
63 fi
64 fi
65}
66
67# logging paths for runlogs and processlog
68runlogpath=$logpath/run/`date +%Y/%m/%d`
69processlogpath=$logpath/processlog
70makedir $runlogpath
71makedir $processlogpath
72processlog=$processlogpath/process`date +%F`.log
73
74makedir $lockpath
75
76
77# function to provide proper logging in a single logfile ($processlog)
78function printprocesslog
79{
80 makedir $processlogpath
81 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "$@ >> $processlog
82}
83
84# function to exit a script properly
85function finish()
86{
87 if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
88 then
89 printprocesslog "DEBUG " `rm -v $lockfile`
90 fi
91 printprocesslog "DEBUG finished "$SOURCEFILEPATH"/"$SCRIPTNAME
92 exit
93}
94
95
96# set checkvalue to ok at the beginning of the scripts
97check="ok"
98
99# setup for jobmanager:
100# log files (can't be defined in script itself, as script can run longer
101# than one day
102jmerrorlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`-error.log
103jmscriptlog=$runlogpath/jobmanager-`whoami`-$HOSTNAME-$AUTOMATIONSETUP-`date +%F`.log
104
105# check if rc-files are available
106if ! ls $steps >/dev/null
107then
108 echo "Can't find steps.rc ($steps)"
109 finish
110fi
111if ! ls $sqlrc >/dev/null
112then
113 echo "Can't find sql.rc ($sqlrc)"
114 finish
115fi
116
117# resetting values for jobmanager
118pno=0
119totalpno=0
120running=0
121queued=0
122runningscript=0
123queuedscript=0
124stillinqueue=0
125
126
127# alias (we cannot check the beginning of the line due to
128# color codes in filldotraw.C)
129alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
130
131
132# in the following the functions, which are needed by several scripts, are
133# defined
134
135# function to check if a process is already locked
136# command line option can be used to execute something, e.g. 'continue'
137function checklock()
138{
139 if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "$SCRIPTNAME"["$$"] "`uname -a` > $lockfile 2>/dev/null
140 then
141 if find $lockfile -amin -5
142 then
143 printprocesslog "INFO lockfile $lockfile exists"
144 else
145 printprocesslog "WARN lockfile $lockfile exists"
146 fi
147 $@
148 exit
149 else
150 printprocesslog "DEBUG created lockfile $lockfile"
151 fi
152}
153
154# print the current status values
155function printstatusvalues()
156{
157 echo "the current values are:"
158 echo " starttime=$starttime"
159 echo " stoptime=$stoptime"
160 echo " availtime=$availtime"
161 echo " returncode=$returncode"
162 echo "-- check: -$check-"
163 echo ""
164}
165
166# get the db-setup from the sql.rc
167function getdbsetup()
168{
169 db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
170 pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
171 us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
172 ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
173# echo "setup: "
174# echo " db: "$db
175# echo " pw: "$pw
176# echo " us: "$us
177# echo " ho: "$ho
178}
179
180# function to send a mysql query
181function sendquery()
182{
183 getdbsetup
184 printprocesslog "DEBUG sendquery QUERY: "$query
185 if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
186 then
187 printprocesslog "ERROR could not query DB "$db" on host "$ho" with user "$us
188 #printprocesslog "ERROR could not query db (program: $program, function sendquery)"
189 #return 1 #why???
190 finish
191 fi
192 if [ "$val" = "NULL" ]
193 then
194 val=
195 fi
196 echo $val
197 return 0
198}
199
200# function to get information from the setupfile $steps
201function getfromsetup()
202{
203 grep $1"[.]"$2":" $steps | grep -v '#' | sed -e "s/$1[.]$2://"
204}
205
206# function to get the needed information from the dependencies-file steps.rc
207function getstepinfo()
208{
209 getdbsetup
210 needs=( `getfromsetup $step "Needs"` )
211 noderestricted=`getfromsetup $step "NodeRestricted"`
212 prims=( `getfromsetup $step "Primaries"` )
213 maintable=`getfromsetup $step "MainTable" | sed -e "s/\ //g"`
214 sort=`getfromsetup $step "SortDirection" | sed -e "s/\ //g"`
215# echo " maintable: "$maintable
216# echo " needs: "${needs[@]}
217# echo " noderestricted: "$noderestricted
218# echo " prims: "${prims[@]}
219}
220
221# function to get the joins needed for the get/set status queries
222function getalljoins()
223{
224 # add table
225 query=$query" "$maintable"Status"
226 # add special join
227 query=$query" "`getfromsetup $maintable "SpecialJoin"`
228 # add join for step unless step is the same as maintable
229 if ! [ "$step" = "$maintable" ]
230 then
231 query=$query" LEFT JOIN "$step"Status USING("${prims[@]}") "
232 fi
233 # add joins for influences or needs
234 for otherstep in ${othersteps[@]}
235 do
236 if ! [ "$otherstep" = "$maintable" ]
237 then
238 query=$query" LEFT JOIN "$otherstep"Status USING("`getfromsetup $otherstep "Primaries"`") "
239 fi
240 done
241}
242
243# function to create the middle part of a query
244# which is identical for the functions getstatus() and gettodo()
245function getstatusquery()
246{
247 # add from which table the information is queried
248 query=$query" FROM "
249 othersteps=${needs[@]}
250 getalljoins
251 # add condition
252 query=$query" WHERE "
253 # add condition for step, i.e. step is not yet done
254 query=$query" ISNULL("$step"Status.fStartTime) "
255 query=$query" AND ISNULL("$step"Status.fStopTime) "
256 query=$query" AND ISNULL("$step"Status.fAvailable) "
257 query=$query" AND ISNULL("$step"Status.fReturnCode) "
258 # add requirement for production host in case it is needed
259 if [ "$1 " != " " ]
260 then
261 query=$query" AND fProductionHostKEY=$2 "
262 fi
263 if ! echo $query | grep UPDATE >/dev/null 2>&1
264 then
265 query=$query" GROUP BY "${prims[@]}
266 fi
267 # add condition for needs, i.e. that step is done
268 for (( k=0 ; k < ${#needs[@]} ; k++ ))
269 do
270 if [ $k -eq 0 ]
271 then
272 query=$query" HAVING "
273 else
274 query=$query" AND "
275 fi
276 query=$query" COUNT(*)=COUNT(IF("
277 query=$query" NOT ISNULL("${needs[$k]}"Status.fStartTime) "
278 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fStopTime) "
279 query=$query" AND NOT ISNULL("${needs[$k]}"Status.fAvailable) "
280 query=$query" AND ISNULL("${needs[$k]}"Status.fReturnCode) "
281 query=$query" , 1, NULL)) "
282 done
283}
284
285# function to get todolist
286# returns the next or the list of next steps
287function gettodo()
288{
289 # reset the variable for the number of the next step
290 process=
291 printprocesslog "DEBUG getting todo for step $step..."
292 getstepinfo
293 # get query
294 query=" SELECT "${prims[@]}
295 getstatusquery $2
296 # order by priority to the the number of the next step to be done
297 query=$query" ORDER BY "$step"Status.fPriority "
298 if [ "$sort" = "" ]
299 then
300 query=$query" DESC "
301 else
302 query=$query" "$sort
303 fi
304 # add limitation in case only one or a limited number of
305 # processes should be executed
306 if [ "$1 " != " " ]
307 then
308 query=$query" limit 0, $1 "
309 fi
310 # print query
311 printprocesslog "DEBUG gettodo for step $step QUERY: "$query
312 # execute query
313 process=`sendquery`
314 #if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
315 #then
316 # printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
317 # finish
318 #fi
319 # get numbers of next step from mysql result
320 if [ "$process" = "" ]
321 then
322 printprocesslog "DEBUG => nothing to do"
323 finish
324 else
325 primaries=( $process )
326 num=`expr ${#primaries[@]} / ${#prims[@]} `
327 fi
328}
329
330# function to get the number of processes which still have to be done
331function getstatus()
332{
333 # reset the variable for the number of steps to be done
334 numproc=0
335 getstepinfo
336 # get query
337 query=" SELECT "${prims[@]}
338 getstatusquery $1
339 # print query
340 printprocesslog "DEBUG getstatus for step $step QUERY: "$query
341 # execute query
342 #numproc=`sendquery `#cannot be done with sendquery, because of row counting
343 if ! numproc=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query " | wc -l`
344 then
345 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
346 echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
347 continue
348 fi
349}
350
351# function to set status of a process in the db
352function setstatus()
353{
354 # remark:
355 # this function does not include the 'Default' flag
356 # for resetting steps
357
358 # for dowebplots (there are steps which have no entry in the DB)
359 if [ "$step" = "no" ]
360 then
361 return
362 fi
363
364 # reset status values
365 starttime=NULL
366 stoptime=NULL
367 availtime=NULL
368 returncode=NULL
369 # evaluate the status values
370 case $@ in
371 start) printprocesslog "DEBUG setstatus start"
372 starttime="Now()"
373 ;;
374 stop) case $check in
375 ok) printprocesslog "DEBUB setstatus stop - ok"
376 starttime=noreset
377 stoptime="Now()"
378 if [ "$processingsite" = "$storagesite" ]
379 then
380 availtime="Now()"
381 fi
382 ;;
383 no) printprocesslog "DEBUG setstatus stop - nothing new"
384 check="ok"
385 ;;
386 *) printprocesslog "DEBUG setstatus stop - failed"
387 starttime=noreset
388 stoptime="Now()"
389 if [ "$processingsite" = "$storagesite" ]
390 then
391 availtime="Now()"
392 fi
393 if [ "$check" == "" ]
394 then
395 returncode=1
396 else
397 returncode=$check
398 fi
399 check="ok"
400 ;;
401 esac
402 ;;
403 *) printprocesslog "ERROR function setstatus got wrong variable"
404 finish
405 ;;
406 esac
407
408 # get
409 getstepinfo
410
411 # get the influences from the steps.rc by evaluating the needs of all steps
412 othersteps=`grep $step $steps | grep -v '#' | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
413
414 # get query
415 query=" UPDATE "
416 getalljoins
417 # set the status values according to the new status of the step
418 query=$query" SET "
419 if ! [ "$starttime" = "noreset" ]
420 then
421 query=$query" "$step"Status.fStartTime=$starttime, "
422 fi
423 query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fAvailable=$availtime"
424 query=$query", "$step"Status.fReturnCode=$returncode , "$step"Status.fProcessingSiteKEY=$sitekey "
425 # set also the status values of the influenced steps
426 for otherstep in $othersteps
427 do
428 query=$query", "$otherstep"Status.fStartTime=NULL "
429 query=$query", "$otherstep"Status.fStopTime=NULL "
430 query=$query", "$otherstep"Status.fAvailable=NULL "
431 query=$query", "$otherstep"Status.fReturnCode=NULL "
432 query=$query", "$otherstep"Status.fProcessingSiteKEY=NULL "
433 done
434 # give the condition for which step the status values have to be set
435 query=$query" WHERE "
436 if [ "$s" = "" ]
437 then
438 s=0
439 fi
440 query=$query" "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
441 for (( j=1 ; j < ${#prims[@]} ; j++ ))
442 do
443 query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
444 done
445 # add additional query to allow for locking in db
446 if [ "$1" = "start" ]
447 then
448 query=$query" AND ISNULL("$step"Status.fStartTime) "
449 fi
450 # add row count to know how many rows have been changed
451 query=$query"; SELECT ROW_COUNT();"
452 # print query
453 printprocesslog "DEBUG setstatus for step $step QUERY: "$query
454 #echo "DEBUG setstatus for step $step QUERY: "$query
455 # execute query
456 numchanged=`sendquery`
457 #echo "numchanged: "$numchanged
458 #if ! numchanged=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
459 #then
460 # printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
461 # finish
462 #fi
463 if [ $numchanged -gt 0 ]
464 then
465 printprocesslog "INFO successful set of status in DB."
466 #echo "INFO successful set of status in DB."
467 else
468 # action may be taken in script using $numchanged
469 printprocesslog "ERROR status in DB was already set by another process "
470 #echo "ERROR status in DB was already set by another process "
471 fi
472}
473
474function getdates()
475{
476 case $1 in
477 # all dates
478 all)
479 printprocesslog "DEBUG getdates case 'all'"
480 dates=( `find $auxdata -mindepth 3 -type d | sort -r | sed "s/\${auxdata_for_sed}//g" | sed -e 's/^\///'` )
481 ;;
482 # certain date
483 [0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9])
484 printprocesslog "DEBUG getdates case certain date: "$1
485 dates=( $1 )
486 ;;
487 # certain number of dates (between 0 and 99 )
488 [0-9][0-9]|[0-9])
489 # get last n nights
490 for (( numdates=1 ; numdates <= $1 ; numdates++ ))
491 do
492 numhours=`echo " 12 + ( $numdates - 1 ) * 24 " | bc -l`
493 dates=( ${dates[@]} `date +%Y/%m/%d --date="-"$numhours"hour"` )
494 done
495 # hour-dependent number of dates
496 if [ "$2" != "" ] && [ "$3" != "" ]
497 then
498 # get current hour
499 hour=`date +%k`
500 if [ $hour -le $2 ] || [ $hour -ge $3 ]
501 then
502 dates=( `date +%Y/%m/%d --date="-12hour"` )
503 fi
504 fi
505 ;;
506 *) # nothing valid given
507 echo "Please give valid options for the function 'getdates()'"
508 finish
509 ;;
510 esac
511}
512
Note: See TracBrowser for help on using the repository browser.