source: trunk/MagicSoft/Mars/datacenter/scripts/sourcefile @ 9592

Last change on this file since 9592 was 9592, checked in by dorner, 11 years ago
*** empty log message ***
  • Property svn:executable set to *
File size: 16.9 KB
Line 
1#!/bin/sh
2#
3# ========================================================================
4#
5# *
6# * This file is part of MARS, the MAGIC Analysis and Reconstruction
7# * Software. It is distributed to you in the hope that it can be a useful
8# * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
9# * It is distributed WITHOUT ANY WARRANTY.
10# *
11# * Permission to use, copy, modify and distribute this software and its
12# * documentation for any purpose is hereby granted without fee,
13# * provided that the above copyright notice appear in all copies and
14# * that both that copyright notice and this permission notice appear
15# * in supporting documentation. It is provided "as is" without express
16# * or implied warranty.
17# *
18#
19#
20#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
21#
22#   Copyright: MAGIC Software Development, 2000-2010
23#
24#
25# ========================================================================
26#
27# This a resource file for the scripts, in which the standard paths and
28# functions, which are needed more often are stored.
29# Only constant variables are stored here, changing variables are stored
30# in datacenter/scripts/setup
31#
32
33# check if script has been started with absolute path
34if ! dirname $0 | grep -E '^/' >/dev/null 2>&1
35then 
36   echo "Please start your script with an absolute path."
37   exit
38fi
39
40if [ "$AUTOMATIONSETUP" = "" ]
41then 
42   echo "Please set the environment variable \$AUTOMATIONSETUP."
43   exit
44fi
45
46# possible solution for $0 problem:
47# ${BASH_SOURCE[0]}
48# but has to be checked and tested more carefully
49# should solve at least problem with login-shell and with source
50source `dirname $0`/setup.$AUTOMATIONSETUP
51
52datetime=`date +%F-%H-%M-%S`
53
54
55# function to make sure that a directory is made
56function makedir()
57{
58   if [ ! -d $@ ]
59   then
60      mkdir -pv $@ 
61      if [ ! -d $@ ]
62      then 
63         if ! [ "$processlog" = "" ]
64         then
65            echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] ERROR could not make dir "$@ >> $processlog
66         else
67            echo "could not make dir "$@
68         fi
69         if ls $lockfile >/dev/null 2>&1
70         then 
71            rm -v $lockfile
72         fi
73         exit
74      fi
75   fi
76}
77
78# logging paths for runlogs and processlog
79runlogpath=$logpath/run/`date +%Y/%m/%d`
80processlogpath=$logpath/processlog
81makedir $runlogpath
82makedir $processlogpath
83processlog=$processlogpath/process`date +%F`.log
84
85makedir $lockpath
86
87
88# function to provide proper logging in a single logfile ($processlog)
89function printprocesslog
90{
91   makedir $processlogpath
92   echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "$@ >> $processlog
93}
94
95# function to exit a script properly
96function finish()
97{
98   if ! [ "$lockfile" = "" ] && ls $lockfile >/dev/null 2>&1
99   then 
100      printprocesslog "INFO " `rm -v $lockfile`
101   fi
102   printprocesslog "INFO finished $0"
103   exit
104}
105
106
107# set checkvalue to ok at the beginning of the scripts
108check="ok"
109
110#failed codes
111#sequence build status
112Fbuildsequ=1
113Fdoexcl=2
114#run process status
115Ftimecorr=3
116Ffillraw=4
117Fsinope=5
118Ffillsinope=6
119Fresetexcl=7
120#sequence process status
121Fwritesequfile=8
122Ffilesavail=9
123Fnoccfile=10
124Fnocacofile=11
125Fmerppcc=12
126Fmerppcaco=13
127Fcallisto=14
128Ffillcalib=15
129Ffillsignal=16
130Fstar=17
131Ffillstar=18
132#dataset process status
133Fwritedatasetfile=19
134Fstardone=20
135Fganymed=21
136Ffillganymed=22
137#again run process status
138FCompmux=26
139Fdowebplots=27
140#again mc run process status
141Fmccallisto=28
142Ffillmccalib=29
143Ffillmcsignal=30
144Fmcstar=31
145Ffillmcstar=32
146Fcorsikasimtel=33
147Fchimp=34
148Fchimpcp=35
149Fctastar=36
150Fctastarcp=37
151FctastereoA=38
152FctastereoB=39
153FctastereoC=40
154FctastereoD=41
155FctastereoE=42
156FctastereoF=43
157FctastereoG=44
158FctastereoH=45
159Fctastereocp=46
160
161# setup for jobmanager:
162#   log files (can't be defined in script itself, as script can run longer
163#   than one day
164jmerrorlog=$runlogpath/jobmanager-error`date +%F`.log
165jmscriptlog=$runlogpath/jobmanager`date +%F`.log
166
167# check if rc-files are available
168if ! ls $steps >/dev/null
169then
170   echo "Can't find steps.rc ($steps)"
171   finish
172fi
173if ! ls $sqlrc >/dev/null
174then
175   echo "Can't find sql.rc ($sqlrc)"
176   finish
177fi
178
179# resetting values
180pno=0
181totalpno=0
182running=0
183queued=0
184runningscript=0
185queuedscript=0
186stillinqueue=0
187
188
189# alias (we cannot check the beginning of the line due to
190# color codes in filldotraw.C)
191alias 'intgrep'='grep -E -o \\\(int\\\)[0-9]+$ | grep -E -o [0-9]+'
192
193
194# in the following the functions, which are needed by several scripts, are
195# defined
196
197# function to check if a process is already locked
198#  command line option can be used to execute something, e.g. 'continue'
199function checklock()
200{
201   if ! echo `date +%F\ %T`" "`whoami`"@"$HOSTNAME" "`basename $0`"["$$"] "`uname -a` > $lockfile 2>/dev/null
202   then 
203      printprocesslog "WARN lockfile $lockfile exists"
204      $@
205      exit
206   else
207      printprocesslog "INFO created lockfile $lockfile"
208   fi
209}
210
211# print the current status values
212function printstatusvalues()
213{
214   echo "the current values are:"
215   echo " starttime=$starttime"
216   echo " stoptime=$stoptime"
217   echo " returncode=$returncode"
218   echo "-- check: -$check-"
219   echo ""
220}
221
222# get the db-setup from the sql.rc
223function getdbsetup()
224{
225   db=`grep Database $sqlrc | grep -v '#' | sed -e 's/Database: //' -e 's/ //g'`
226   pw=`grep Password $sqlrc | grep -v '#' | sed -e 's/Password: //' -e 's/ //g'`
227   us=`grep User $sqlrc | grep -v '#' | sed -e 's/User: //' -e 's/ //g'`
228   ho=`grep URL $sqlrc | grep -v '#' | sed -e 's/ //g' -e 's/URL:mysql:\/\///'`
229#   echo "setup: "
230#   echo " db: "$db
231#   echo " pw: "$pw
232#   echo " us: "$us
233#   echo " ho: "$ho
234}
235
236# function to get the needed information from the dependencies-file steps.rc
237function getstepinfo()
238{
239   getdbsetup
240   needs=`grep "$step[.]Needs:" $steps | grep -v '#' | sed -e "s/$step[.]Needs://"`
241   noderestricted=`grep "$step[.]NodeRestricted:" $steps | grep -v '#' | sed -e "s/$step[.]NodeRestricted://" -e 's/ //g'`
242   prims=( `grep "$step[.]Primaries:" $steps | grep -v '#' | sed -e "s/$step[.]Primaries://"` )
243#   echo " needs: $needs"
244#   echo " noderestricted: $noderestricted"
245#   echo " prims: ${prims[@]}"
246}
247
248# function to get the primaries of a step from the dependencies-file steps.rc
249function getprimary()
250{
251   getdbsetup
252   grep $@"[.]Primaries:" $steps | grep -v '#' | sed -e "s/$@[.]Primaries://"
253}
254
255# function to get the join of a step
256#  (normal join but using different primaries)
257#  from the dependencies-file steps.rc
258function getjoin()
259{
260   getdbsetup
261   grep $@"[.]Join:" $steps | grep -v '#' | sed -e "s/$@[.]Join://"
262}
263
264# function to get the special join of a step
265#  (i.e. join with a different table)
266#  from the dependencies-file steps.rc
267function getspecialjoin()
268{
269   getdbsetup
270   grep $@"[.]SpecialJoin:" $steps | grep -v '#' | sed -e "s/$@[.]SpecialJoin://"
271}
272
273# function to get the needs of a step from the dependencies-file steps.rc
274function getneeds()
275{
276   getdbsetup
277   grep $@"[.]Needs:" $steps | sed -e "s/$@[.]Needs://"
278}
279
280# function to create the middle part of a query
281#  which is identical for the functions getstatus() and gettodo()
282function middlepartofquery()
283{
284   # add from which table the information is queried
285   query=$query" FROM "$step"Status "
286   # add the joins to the tables in which the status of the preceding steps is stored
287   for need in $needs
288   do
289      needprims=( `getprimary $need` )
290      # make sure that the correct joins are used
291      # in case the primaries of the step and the need are
292      #   the same, the tables are join with these primaries
293      # otherwise it is checked whether there has to be a
294      #   join with different primaries (indicated as 'Join'
295      #   in the steps.rc
296      # 'SpecialJoins' do not have to be taken into account here
297      if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ]
298      then 
299         query=$query" LEFT JOIN "$need"Status USING (${prims[@]}) "
300      else 
301         stdjoin=`getjoin $need`
302         if ! [ "$stdjoin" = "" ]
303         then
304            query=$query" "$stdjoin" "
305         fi
306      fi
307   done
308   # add condition
309   query=$query" WHERE "
310   # add condition for the status of the peceding steps
311   counter=0
312   for need in $needs
313   do
314      if [ $counter -gt 0 ]
315      then
316         query=$query" AND "
317      fi
318      needprims=( `getprimary $need` )
319      # in case the primaries of the tables agree
320      #   or there is just a normal 'Join' needed
321      #   only the condition is added to the query
322      # for tables which need a join with a different
323      #   table, a special query is added using the
324      #   'SpecialJoin' from the steps.rc
325      if [ "`echo ${needprims[@]}`" == "`echo ${prims[@]}`" ] || ! [ "$stdjoin" = "" ]
326      then 
327         query=$query" NOT ISNULL("$need"Status.fStartTime) AND "
328         query=$query" NOT ISNULL("$need"Status.fStopTime) AND "
329         query=$query" ISNULL("$need"Status.fReturnCode) "
330      else 
331         query=$query" (SELECT COUNT(*) FROM "$need"Status "
332         query=$query" "`getspecialjoin $need`" "
333         query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
334         for (( j=1 ; j < ${#prims[@]} ; j++ ))
335         do
336            query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
337         done
338         query=$query") = (SELECT COUNT(*) FROM "$need"Status "
339         query=$query" "`getspecialjoin $need`" "
340         query=$query" WHERE "$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[0]} | sed -e 's/,//g'`
341         for (( j=1 ; j < ${#prims[@]} ; j++ ))
342         do
343            query=$query" AND "$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`"="$step"Status."`echo ${prims[$j]} | sed -e 's/,//g'`
344         done
345         query=$query" AND NOT ISNULL(fStartTime) "
346         query=$query" AND NOT ISNULL(fStopTime) "
347         query=$query" AND ISNULL(fReturnCode)) "
348      fi
349      counter=`echo $counter + 1 | bc -l`
350   done
351   # add condition for the status of the step itself
352   if [ $counter -gt 0 ]
353   then
354      query=$query" AND "
355   fi
356   query=$query" ISNULL("$step"Status.fStartTime) "
357   query=$query" AND ISNULL("$step"Status.fStopTime) "
358   query=$query" AND ISNULL("$step"Status.fReturnCode) "
359}
360
361# function to get todolist
362#   returns the next or the list of next steps
363function gettodo()
364{
365   # reset the variable for the number of the next step
366   process=
367   printprocesslog "INFO getting todo..."
368   getstepinfo
369   # get query
370   query=" SELECT "`echo ${prims[0]} | sed -e 's/,//g'`
371   for (( j=1 ; j < ${#prims[@]} ; j++ ))
372   do
373      query=$query", "`echo ${prims[$j]} | sed -e 's/,//g'`
374   done
375   # get middle par of query
376   middlepartofquery
377   # add requirement for production host in case it is needed
378   if [ "$2 " != " " ]
379   then 
380      query=$query" AND fProductionHostKEY=$2 "
381   fi
382   # order by priority to the the number of the next step to be done
383   query=$query" ORDER BY "$step"Status.fPriority desc "
384   # add limitation in case only one or a limited number of
385   #  processes should be executed
386   if [ "$1 " != " " ]
387   then 
388      query=$query" limit 0, $1 "
389   fi
390   # print query
391   #echo " gettodo QUERY: "$query
392   printprocesslog "INFO gettodo QUERY: "$query
393   # execute query
394   if ! process=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
395   then
396      printprocesslog "ERROR could not query processes from db (program: $program, function gettodo)"
397      finish
398   fi
399   # get numbers of next step from mysql result
400   if [ "$process" = "" ]
401   then
402      printprocesslog "INFO => nothing to do"
403      finish
404   else
405      primaries=( $process )
406      num=`expr ${#primaries[@]} / ${#prims[@]} `
407   fi
408}
409
410# function to get the number of processes which still have to be done
411function getstatus()
412{
413   # reset the variable for the number of steps to be done
414   numproc=
415   getstepinfo
416   # get query
417   query=" SELECT COUNT(*), 1 " # the 1 is just for grouping
418   # get middle part of query
419   middlepartofquery
420   # add requirement for production host in case it is needed
421   if [ "$1 " != " " ]
422   then 
423      query=$query" AND fProductionHostKEY=$1 "
424   fi
425   # group by an 'artifical' column to get the number of lines
426   query=$query" GROUP BY 2 "
427   # printing query
428   #echo " getstatus QUERY: "$query
429   printprocesslog "INFO getstatus QUERY: "$query
430   # execute query
431   if ! numprocs=( `mysql -s -u $us --password=$pw --host=$ho $db -e " $query "` )
432   then
433      printprocesslog "ERROR could not query number of processes from db (program: $program, function getstatus)"
434      echo `date +%F\ %T`" ERROR could not query number of processes from db (program: $program, function getstatus)"
435      continue
436   fi
437   # get number of processes from mysql result
438   if [ "${numprocs[1]}" = "" ]
439   then
440      numproc=0
441   else
442      numproc=${numprocs[0]}
443   fi
444}
445
446# function to set status of a process in the db
447function setstatus()
448{
449   # remark:
450   # this function does not include the 'Default' flag
451   # for resetting steps
452
453   # for dowebplots (there are steps which have no entry in the DB)
454   if [ "$step" = "no" ]
455   then
456      return
457   fi
458   
459   # reset status values
460   starttime=NULL
461   stoptime=NULL
462   returncode=NULL
463   # evaluate the status values
464   case $@ in
465      start)   printprocesslog "INFO setstatus start"
466               starttime="Now()"
467               ;;
468       stop)   case $check in
469                  ok)  printprocesslog "INFO setstatus stop - ok"
470                       starttime=noreset
471                       stoptime="Now()"
472                       ;;
473                  no)  printprocesslog "INFO setstatus stop - nothing new"
474                       check="ok"
475                       ;;
476                   *)  printprocesslog "INFO setstatus stop - failed"
477                       starttime=noreset
478                       stoptime="Now()"
479                       if [ "$check" == "" ]
480                       then
481                          returncode=1
482                       else
483                          returncode=$check
484                       fi
485                       check="ok"
486                       ;;
487               esac
488               ;;
489          *)   printprocesslog "ERROR function setstatus got wrong variable"
490               finish
491               ;;
492   esac
493   # get
494   getstepinfo
495   
496   # get the influences from the steps.rc by evaluating the needs of all steps
497   influences=`grep $step $steps | grep "Needs" | grep -v "$step[.]Needs" | cut -d'.' -f1`
498   
499   # get query
500   query=" UPDATE "$step"Status "
501   # add joins to the influenced tables
502   for influence in $influences
503   do
504      # make sure that the right join is done
505      #   in case the is a 'Join' given for the
506      #   influenced step, this is added
507      # otherwise it is checked whether the executed step
508      #   itself needs a 'Join' (i.e. has different primaries)
509      #   if yes, the tables are joined with the primaries of the influenced step
510      #   else with the primaries of the executed step itself
511      stdjoin=`getjoin $influence`
512      if ! [ "$stdjoin" = "" ]
513      then 
514         query=$query" "$stdjoin" "
515      else
516         stdjoin2=`getjoin $step`
517         if [ "$stdjoin2" == "" ]
518         then
519            query=$query" LEFT JOIN "$influence"Status USING("`getprimary $influence`") "
520         else
521            query=$query" LEFT JOIN "$influence"Status USING("`getprimary $step`") "
522         fi
523      fi
524   done
525   # set the status values according to the new status of the step
526   query=$query" SET "
527   if ! [ "$starttime" = "noreset" ]
528   then
529      query=$query" "$step"Status.fStartTime=$starttime, "
530   fi
531   query=$query" "$step"Status.fStopTime=$stoptime, "$step"Status.fReturnCode=$returncode "
532   # set also the status values of the influenced steps
533   for influence in $influences
534   do
535      query=$query", "$influence"Status.fStartTime=NULL "
536      query=$query", "$influence"Status.fStopTime=NULL "
537      query=$query", "$influence"Status.fReturnCode=NULL "
538   done
539   # give the condition for which step the status values have to be set
540   query=$query" WHERE "
541   if [ "$s" = "" ]
542   then
543      s=0
544   fi
545   query=$query" "`echo ${prims[0]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}]}'"
546   for (( j=1 ; j < ${#prims[@]} ; j++ ))
547   do
548      query=$query" AND "`echo ${prims[$j]} | sed -e 's/,//g'`"='${primaries[$s*${#prims[@]}+$j]}' "
549   done   
550   # print query
551   #echo " setstatus QUERY: "$query
552   printprocesslog "INFO setstatus QUERY: "$query
553   # execute query
554   if ! mysql -s -u $us --password=$pw --host=$ho $db -e " $query "
555   then
556      printprocesslog "ERROR could not set status in db (program: $program, function setstatus)"
557      finish
558   fi
559}
560
561# function to send a mysql query
562function sendquery()
563{
564   getdbsetup
565   printprocesslog "INFO sendquery QUERY: "$query
566   if ! val=`mysql -s -u $us --password=$pw --host=$ho $db -e " $query "`
567   then
568      printprocesslog "ERROR could not query db (program: $program, function sendquery)"
569      return 1
570   fi
571   if [ "$val" = "NULL" ]
572   then
573      val=
574   fi
575   echo $val
576   return 0
577}
578
Note: See TracBrowser for help on using the repository browser.