source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile @ 9586

Last change on this file since 9586 was 9586, checked in by dorner, 11 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 15.0 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22#   Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then 
36   echo "Please start your script with an absolute path."
37   exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then 
42   echo "Please set the environment variable \$AUTOMATIONSETUP."
43   exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58   if [ ! -d $@ ]
59   then
60      mkdir -pv $@ 
61      if [ ! -d $@ ]
62      then 
63         if ! [ "$processlog" = "" ]
64         then
65            echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66         else
67            echo "could not make dir "$@
68         fi
69         if ls $lockfile >/dev/null 2>&1
70         then 
71            rm -v $lockfile
72         fi
73         exit
74      fi
75   fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91   makedir $processlogpath
92   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99   then 
100      printprocesslog "INFO " `rm -v $lockfile`
101   fi
102   printprocesslog "INFO finished $0"
103   exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110#failed codes
111#sequence build status
112Fbuildsequ=1
113Fdoexcl=2
114#run process status
115Ftimecorr=3
116Ffillraw=4
117Fsinope=5
118Ffillsinope=6
119Fresetexcl=7
120#sequence process status
121Fwritesequfile=8
122Ffilesavail=9
123Fnoccfile=10
124Fnocacofile=11
125Fmerppcc=12
126Fmerppcaco=13
127Fcallisto=14
128Ffillcalib=15
129Ffillsignal=16
130Fstar=17
131Ffillstar=18
132#dataset process status
133Fwritedatasetfile=19
134Fstardone=20
135Fganymed=21
136Ffillganymed=22
137#again run process status
138FCompmux=26
139Fdowebplots=27
140#again mc run process status
141Fmccallisto=28
142Ffillmccalib=29
143Ffillmcsignal=30
144Fmcstar=31
145Ffillmcstar=32
146Fcorsikasimtel=33
147Fchimp=34
148Fchimpcp=35
149Fctastar=36
150Fctastarcp=37
151FctastereoA=38
152FctastereoB=39
153FctastereoC=40
154FctastereoD=41
155FctastereoE=42
156FctastereoF=43
157FctastereoG=44
158FctastereoH=45
159Fctastereocp=46
160
161# setup for jobmanager:
162#   log files (can't be defined in script itself, as script can run longer
163#   than one day
164jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
165jmscriptlog=$runlogpath/jobmanager`date +%F`.log
166
167# check if rc-files are available
168if ! ls $steps >/dev/null
169then
170   echo "Can't find steps.rc ($steps)"
171   finish
172fi
173if ! ls $sqlrc >/dev/null
174then
175   echo "Can't find sql.rc ($sqlrc)"
176   finish
177fi
178
179# resetting values
180pno=0
181totalpno=0
182running=0
183queued=0
184runningscript=0
185queuedscript=0
186stillinqueue=0
187
188
189# alias (we cannot check the beginning of the line due to
190# color codes in filldotraw.C)
191alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
192
193
194# in the following the functions, which are needed by several scripts, are
195# defined
196
197# function to check if a process is already locked
198#  command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201   if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202   then 
203      printprocesslog "WARN lockfile $lockfile exists"
204      $@
205      exit
206   else
207      printprocesslog "INFO created lockfile $lockfile"
208   fi
209}
210
211# print the current status values
212function printstatusvalues()
213{
214   echo "the current values are:"
215   echo " starttime=$starttime"
216   echo " stoptime=$stoptime"
217   echo " returncode=$returncode"
218   echo "-- check: -$check-"
219   echo ""
220}
221
222# get the db-setup from the sql.rc
223function getdbsetup()
224{
225   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
226   pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
227   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
228   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
229#   echo "setup: "
230#   echo " db: "$db
231#   echo " pw: "$pw
232#   echo " us: "$us
233#   echo " ho: "$ho
234}
235
236# function to get the needed information from the dependencies-file steps.rc
237function getstepinfo()
238{
239   getdbsetup
240   needs=`grep "$step[.]Needs:" $steps | sed -e "s/$step[.]Needs://"`
241   noderestricted=`grep "$step[.]NodeRestricted:" $steps | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
242   prims=( `grep "$step[.]Primaries:" $steps | sed -e "s/$step[.]Primaries://"` )
243#   echo " needs: $needs"
244#   echo " noderestricted: $noderestricted"
245#   echo " prims: ${prims[@]}"
246}
247
248# function to get the primaries of a step from the dependencies-file steps.rc
249function getprimary()
250{
251   getdbsetup
252   grep $@"[.]Primaries:" $steps | sed -e "s/$@[.]Primaries://"
253}
254
255# function to get the join of a step from the dependencies-file steps.rc
256function getjoin()
257{
258   getdbsetup
259   grep $@"[.]Join:" $steps | sed -e "s/$@[.]Join://"
260}
261
262# function to create the middle part of a query
263#  which is identical for the functions getstatus() and gettodo()
264function middlepartofquery()
265{
266   # add from which table the information is queried
267   query=$query" FROM "$step"Status "
268   # add the joins to the tables in which the status of the preceding steps is stored
269   for need in $needs
270   do
271      needprims=( `getprimary $need` )
272      if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
273      then 
274         query=$query" LEFT JOIN "$need"Status USING (${prims[@]}) "
275      fi
276   done
277   # add condition
278   query=$query" WHERE "
279   # add condition for the status of the peceding steps
280   counter=0
281   for need in $needs
282   do
283      if [ $counter -gt 0 ]
284      then
285         query=$query" AND "
286      fi
287      needprims=( `getprimary $need` )
288      # in case the primaries of the tables agree
289      #   only the condition is given
290      # for tables with differing primaries a special query
291      #   is needed
292      if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
293      then 
294         query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
295         query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
296         query=$query" ISNULL("$need"Status.fReturnCode) "
297      else 
298         query=$query" (SELECT COUNT(*) FROM "$need"Status "
299         query=$query" "`getjoin $need`" "
300         query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
301         for (( j=1 ; j < ${#prims[@]} ; j++ ))
302         do
303            query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
304         done
305         query=$query") = (SELECT COUNT(*) FROM "$need"Status "
306         query=$query" "`getjoin $need`" "
307         query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
308         for (( j=1 ; j < ${#prims[@]} ; j++ ))
309         do
310            query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
311         done
312         query=$query" AND NOT ISNULL(fStartTime) "
313         query=$query" AND NOT ISNULL(fStopTime) "
314         query=$query" AND ISNULL(fReturnCode)) "
315      fi
316      counter=`echo $counter + 1 | bc -l`
317   done
318   # add condition for the status of the step itself
319   if [ $counter -gt 0 ]
320   then
321      query=$query" AND "
322   fi
323   query=$query" ISNULL("$step"Status.fStartTime) "
324   query=$query" AND ISNULL("$step"Status.fStopTime) "
325   query=$query" AND ISNULL("$step"Status.fReturnCode) "
326}
327
328# function to get todolist
329#   returns the next or the list of next steps
330function gettodo()
331{
332   # reset the variable for the number of the next step
333   process=
334   printprocesslog "INFO getting todo..."
335   getstepinfo
336   # get query
337   query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
338   for (( j=1 ; j < ${#prims[@]} ; j++ ))
339   do
340      query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
341   done
342   # get middle par of query
343   middlepartofquery
344   # add requirement for production host in case it is needed
345   if [ "$2 " != " " ]
346   then 
347      query=$query" AND fProductionHostKEY=$2 "
348   fi
349   # order by priority to the the number of the next step to be done
350   query=$query" ORDER BY "$step"Status.fPriority desc "
351   # add limitation in case only one or a limited number of
352   #  processes should be executed
353   if [ "$1 " != " " ]
354   then 
355      query=$query" limit 0, $1 "
356   fi
357   # print query
358   echo " gettodo QUERY: "$query
359   printprocesslog "INFO gettodo QUERY: "$query
360   # execute query
361   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
362   then
363      printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
364      finish
365   fi
366   # get numbers of next step from mysql result
367   if [ "$process" = "" ]
368   then
369      printprocesslog "INFO => nothing to do"
370      finish
371   else
372      primaries=( $process )
373      num=`expr ${#primaries[@]} / ${#prims[@]} `
374   fi
375}
376
377# function to get the number of processes which still have to be done
378function getstatus()
379{
380   # reset the variable for the number of steps to be done
381   numproc=
382   getstepinfo
383   # get query
384   query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
385   # get middle part of query
386   middlepartofquery
387   # add requirement for production host in case it is needed
388   if [ "$1 " != " " ]
389   then 
390      query=$query" AND fProductionHostKEY=$1 "
391   fi
392   # group by an 'artifical' column to get the number of lines
393   query=$query" GROUP BY 2 "
394   # printing query
395   echo " getstatus QUERY: "$query
396   printprocesslog "INFO getstatus QUERY: "$query
397   # execute query
398   if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
399   then
400      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
401      echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
402      continue
403   fi
404   # get number of processes from mysql result
405   if [ "${numprocs[1]}" = "" ]
406   then
407      numproc=0
408   else
409      numproc=${numprocs[0]}
410   fi
411}
412
413# function to set status of a process in the db
414function setstatus()
415{
416   # remark:
417   # this function does not include the 'Default' flag
418   # for resetting steps
419
420   # for dowebplots (there are steps which have no entry in the DB)
421   if [ "$step" = "no" ]
422   then
423      return
424   fi
425   
426   # reset status values
427   starttime=NULL
428   stoptime=NULL
429   returncode=NULL
430   # evaluate the status values
431   case $@ in
432      start)   printprocesslog "INFO setstatus start"
433               starttime="Now()"
434               ;;
435       stop)   case $check in
436                  ok)  printprocesslog "INFO setstatus stop - ok"
437                       starttime=noreset
438                       stoptime="Now()"
439                       ;;
440                  no)  printprocesslog "INFO setstatus stop - nothing new"
441                       check="ok"
442                       ;;
443                   *)  printprocesslog "INFO setstatus stop - failed"
444                       starttime=noreset
445                       stoptime="Now()"
446                       if [ "$check" == "" ]
447                       then
448                          returncode=1
449                       else
450                          returncode=$check
451                       fi
452                       check="ok"
453                       ;;
454               esac
455               ;;
456          *)   printprocesslog "ERROR function setstatus got wrong variable"
457               finish
458               ;;
459   esac
460   # get
461   getstepinfo
462   
463   # get the influences from the steps.rc by evaluating the needs of all steps
464   influences=`grep $step $rc | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
465   
466   # get query
467   query=" UPDATE "$step"Status "
468   # add joins to the influenced tables
469   for influence in $influences
470   do
471      query=$query" LEFT JOIN $influence USING("`getprimary $influence`") "
472      specialjoin=`getjoin $influence`
473      if ! [ "$specialjoin" = "" ]
474      then 
475         query=$query$specialjoin
476      fi
477   done
478   # set the status values according to the new status of the step
479   query=$query" SET "
480   if ! [ "$starttime" = "noreset" ]
481   then
482      query=$query" "$step"Status.fStartTime=$starttime, "
483   fi
484   query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
485   # set also the status values of the influenced steps
486   for influence in $influences
487   do
488      query=$query", "$influence"Status.fStartTime=NULL "
489      query=$query", "$influence"Status.fStopTime=NULL "
490      query=$query", "$influence"Status.fReturnCode=NULL "
491   done
492   # give the condition for which step the status values have to be set
493   query=$query" WHERE "
494   if [ "$s" = "" ]
495   then
496      s=0
497   fi
498   query=$query" "`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
499   for (( j=1 ; j < ${#prims[@]} ; j++ ))
500   do
501      query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
502   done   
503   # print query
504   echo " setstatus QUERY: "$query
505   printprocesslog "INFO setstatus QUERY: "$query
506   # execute query
507   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
508   then
509      printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
510      finish
511   fi
512}
513
514# function to send a mysql query
515function sendquery()
516{
517   getdbsetup
518   printprocesslog "INFO sendquery QUERY: "$query
519   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
520   then
521      printprocesslog "ERROR could not query db (program: $program, function sendquery)"
522      return 1
523   fi
524   if [ "$val" = "NULL" ]
525   then
526      val=
527   fi
528   echo $val
529   return 0
530}
531
Note: See TracBrowser for help on using the repository browser.