source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile@ 7944

Last change on this file since 7944 was 7944, checked in by Daniela Dorner, 19 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 11.7 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20# Author(s): Daniela Dorner 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22# Copyright: MAGIC Software Development, 2000-2006
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33source `dirname $0`/setup
34
35mars=`dirname $0 | sed -e 's/\/datacenter\/scripts//'`
36macrospath=$mars/datacenter/macros
37scriptspath=$mars/datacenter/scripts
38
39webpath=/www/htdocs/datacenter
40
41runlogpath=$logpath/run/`date +%Y/%m/%d`
42makedir $runlogpath
43processlog=$runlogpath/process`date +%F`.log
44
45datetime=`date +%F-%H-%M-%S`
46
47check="ok"
48
49#failed codes
50#sequence build status
51Fbuildsequ=1
52Fdoexcl=2
53#run process status
54Ftimecorr=3
55Ffillraw=4
56Fsinope=5
57Ffillsinope=6
58Fresetexcl=7
59#sequence process status
60Fwritesequfile=8
61Ffilesavail=9
62Fnoccfile=10
63Fnocacofile=11
64Fmerppcc=12
65Fmerppcaco=13
66Fcallisto=14
67Ffillcalib=15
68Ffillsignal=16
69Fstar=17
70Ffillstar=18
71#dataset process status
72Fstardone=19
73Fganymed=20
74Ffillganymed=21
75
76#
77# setup for jobmanager
78#
79#log files (can't be defined in script itself, as script can run longer
80#than one day
81jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
82jmscriptlog=$runlogpath/jobmanager`date +%F`.log
83steps=$mars/steps.rc
84# resetting values
85pno=0
86totalpno=0
87running=0
88queued=0
89runningscript=0
90queuedscript=0
91stillinqueue=0
92
93function setzero()
94{
95 val=$@
96 if [ "$val" == "" ]
97 then
98 test=`condor_q -global`
99 if [ "$test" == "" ]
100 then
101 echo $max
102 else
103 echo 0
104 fi
105 else
106 echo $val
107 fi
108}
109
110
111
112# in the following the function, which are needed by several scripts, are
113# defined
114
115function printprocesslog
116{
117 makedir $processlogpath
118 echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
119}
120
121# function to exit a script properly
122function finish()
123{
124 rm -v $todofile
125 rm -v $lockfile
126 date
127 printprocesslog "INFO finished $program"
128 exit
129}
130
131# function to do continue in a loop and produce according logging
132function cont()
133{
134 date
135 echo ""
136 continue
137}
138
139# function to make sure that a directory is made
140function makedir()
141{
142 if [ ! -d $@ ]
143 then
144 mkdir -pv $@
145 if [ ! -d $@ ]
146 then
147 echo "could not make dir "$@
148 finish
149 fi
150 fi
151}
152
153# function to check if a process is already locked
154function checklock()
155{
156 date > $lockfile
157 checklock0=$?
158 case $checklock0 in
159 0) echo " checklock0=$checklock0 -> continue " ;;
160 1) echo " checklock0=$checklock0 -> file $lockfile exists -> exit"
161 date
162 exit;;
163 *) echo " checklock0=$checklock0 -> something went completely wrong" ;;
164 esac
165}
166
167# function calling the macro, which is producing the todo-list
168function getdolistroot()
169{
170 datetime=`date +%F-%H-%M-%S`
171 year=`date +%Y`
172 date=NULL
173
174 getstatuslogpath=$logpath/getstatus/$program/$year
175 getstatuslog=$getstatuslogpath/getstatus-$table-$column-$datetime.log
176 makedir $getstatuslogpath
177
178 # get todo list
179 echo "getting todo list..."
180 check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
181
182 case $check0 in
183 1) echo " check0=$check0 -> everything ok, got todo list -> run $program";;
184 *) echo " check0=$check0 -> ERROR -> could not get todo list -> exit"
185 finish ;;
186 esac
187
188}
189
190function resetstatusvalues()
191{
192 statustime=NULL
193 starttime=NULL
194 returncode=NULL
195 failedcode=NULL
196 failedcodeadd=NULL
197 failedtime=NULL
198}
199
200function printstatusvalues()
201{
202 echo "the current values are:"
203 echo " statustime=$statustime"
204 echo " starttime=$starttime"
205 echo " returncode=$returncode"
206 echo " failedcode=$failedcode"
207 echo " failedcodeadd=$failedcodeadd"
208 echo " failedtime=$failedtime"
209 echo "-- check: -$check-"
210 echo ""
211}
212
213# function evaluating the statusvalues
214function evalstatus()
215{
216 case $@ in
217 start) echo "setstatus start"
218 starttime="Now()"
219 ;;
220 stop) echo "setstatus stop"
221 case $check in
222 ok) echo " ok"
223 statustime="Now()"
224 ;;
225 no) echo "nothing new"
226 check="ok"
227 ;;
228 *) echo " failed"
229 starttime=noreset
230 returncode=$check
231 failedcode=$com
232 if ! [ "$comadd" = "" ]
233 then
234 failedcodeadd=$comadd
235 fi
236 failedtime="Now()"
237 check="ok"
238 ;;
239 esac
240 ;;
241 *) echo "error -> exit"
242 printprocesslog "ERROR function evalstatus got wrong variable"
243 finish
244 ;;
245 esac
246}
247
248# function calling the macro to set the status, after a process has finished
249function setstatusroot()
250{
251 # set status values
252 resetstatusvalues
253 evalstatus $@
254
255# printstatusvalues
256 # set status
257 setstatuslogpath=$logpath/setstatus/$program/$var1
258 makedir $setstatuslogpath
259 setstatuslog=$setstatuslogpath/setstatus-$@-$program-$var2.log
260 checkstatus=`root -q -b $macrospath/setstatus.C+\("\"$var2\""\,"\"$table\""\,"\"$column\""\,"\"$statustime\""\,"\"$returncode\""\,"\"$failedcode\""\,"\"$failedcodeadd\""\,"\"$starttime\""\,"\"$failedtime\""\) | tee $setstatuslog | grep int | sed -e 's/(int)//'`
261 case $checkstatus in
262 1) echo " checkstatus=$checkstatus -> everything ok, status has been set";;
263 *) echo " checkstatus=$checkstatus -> ERROR -> step could not be set -> exit"
264 finish ;;
265 esac
266}
267
268function getdbsetup()
269{
270 db=`grep Database $mars/sql.rc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
271 pw=`grep Password $mars/sql.rc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
272 us=`grep User $mars/sql.rc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
273# echo "setup: "
274# echo " db: "$db
275# echo " pw: "$pw
276# echo " us: "$us
277}
278
279function getstepinfo()
280{
281 table=`grep "$column:" $steps | sed -e "s/[.]$column://" -e 's/#//' -e 's/ //g'`
282 coltab=`grep "$column:" $steps | sed -e 's/://' -e 's/#//' -e 's/ //g'`
283 needs=`grep "$coltab[.]Needs:" $steps | sed -e "s/$coltab[.]Needs://"`
284 influences=`grep "$coltab[.]Influences:" $steps | sed -e "s/$coltab[.]Influences://"`
285 primary=`grep "$table[.]Primary:" $steps | sed -e "s/$table[.]Primary://"`
286# echo " column $column - table $table - coltab $coltab"
287# echo " needs: $needs"
288# echo " influences: $influences"
289# echo " primary: $primary"
290}
291
292# function to get todolist
293function getdolist()
294{
295 echo "getting todolist..."
296 getdbsetup
297 getstepinfo
298 # get query
299 query=" select $primary from $table where "
300 if ! echo $needs | grep '#' > /dev/null
301 then
302 for need in $needs
303 do
304 query=$query" not isnull($need) and"
305 done
306 fi
307 query=$query" isnull($column) "
308 query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
309 query=$query" order by $primary desc "
310# echo " QUERY: "$query
311 if ! process=`mysql -s -u $us --password=$pw --host=hercules $db -e " $query "`
312 then
313 echo "ERROR could not query processes from db -> exit"
314 printprocesslog "ERROR could not query processes from db (program: $program, function getdolist)"
315 finish
316 fi
317
318 if [ "$process" = "" ]
319 then
320 echo " => nothing to do"
321 finish
322 else
323 todofile=$listpath/ToDo-$table-$column.txt
324
325 if ls $todofile > /dev/null 2>&1
326 then
327 echo "$todofile exists already"
328 printprocesslog "WARN $todofile exists already (program: $program, function getdolist)"
329 finish
330 fi
331 echo "found processes, writing todofile..."
332 echo $process > $todofile
333 fi
334}
335
336# function to get todo (process)
337function gettodo()
338{
339 process=
340 echo "getting todo..."
341 getdbsetup
342 getstepinfo
343 # get query
344 query=" select $primary from $table where "
345 if ! echo $needs | grep '#' > /dev/null
346 then
347 for need in $needs
348 do
349 query=$query" not isnull($need) and"
350 done
351 fi
352 query=$query" isnull($column) "
353 query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
354 query=$query" order by $primary desc "
355 query=$query" limit 0, 1 "
356# echo " QUERY: "$query
357 if ! process=`mysql -s -u $us --password=$pw --host=hercules $db -e " $query "`
358 then
359 echo "ERROR could not query process from db -> exit"
360 printprocesslog "ERROR could not query process from db (program: $program, function gettodo)"
361 finish
362 fi
363
364 if [ "$process" = "" ]
365 then
366 echo " => nothing to do -> exit"
367 finish
368 fi
369}
370
371# function to get the number of processes which still have to be done
372function getstatus()
373{
374 printprocesslog "this is a test"`date`
375 numproc=
376# echo "getting status..."
377 getdbsetup
378 column=${scriptscolname[$i]}
379 getstepinfo
380 # get query
381 query=" select count(*) from $table where "
382 if ! echo $needs | grep '#' > /dev/null
383 then
384 for need in $needs
385 do
386 query=$query" not isnull($need) and"
387 done
388 fi
389 query=$query" isnull($column) "
390 query=$query" and isnull(fStartTime) and isnull(fFailedTime) and isnull(fFailedCode) and isnull(fFailedCodeAdd) and isnull(fReturnCode) "
391 query=$query" group by $column "
392# echo " QUERY: "$query
393 if ! numproc=`mysql -s -u $us --password=$pw --host=hercules $db -e " $query "`
394 then
395 echo "ERROR could not query number of todo proceses from db -> continue"
396 printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
397 cont
398 fi
399}
400
401# function to set status of a process in the db
402function setstatus()
403{
404 resetstatusvalues
405 evalstatus $@
406 getdbsetup
407 getstepinfo
408 # get query
409 reset=`grep "$coltab[.]Reset:" $steps | sed -e "s/$coltab[.]Reset://" -e 's/ //g'`
410 if [ "$reset" = "no" ]
411 then
412 echo "YOU CAN'T RESET $column for $var2!!!"
413 printprocesslog "ERROR you can't reset $column for $var2"
414 finish
415 fi
416 query=" update $table set $column=$statustime"
417 if ! echo $influences | grep '#' > /dev/null
418 then
419 for influence in $influences
420 do
421 query=$query", $influence=NULL"
422 done
423 fi
424 query=$query", fStartTime=$starttime, fFailedTime=$failedtime, fFailedCode=$failedcode, fFailedCodeAdd=$failedcodeadd, fReturnCode=$returncode "
425 query=$query" where $primary=$var2"
426 echo " QUERY: "$query
427 if ! mysql -s -u $us --password=$pw --host=hercules $db -e " $query "
428 then
429 echo "ERROR could not insert status into db -> exit"
430 printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
431 finish
432 fi
433
434}
Note: See TracBrowser for help on using the repository browser.