Changeset 9081 for trunk/MagicSoft/Mars/datacenter/scripts

Timestamp:

08/04/08 13:41:17 (16 years ago)

Author:

snruegam

Message:

*** empty log message ***

Location:

trunk/MagicSoft/Mars/datacenter/scripts

Files:

• cutslices (modified) (3 diffs)
• movingrawfiles (modified) (5 diffs)
• setup (modified) (5 diffs)

trunk/MagicSoft/Mars/datacenter/scripts/cutslices

@@ -20 +20 @@
 #   Author(s): Stefan Ruegamer 12/2007 <mailto:snruegam@astro.uni-wuerzburg.de>
 #
-#   Copyright: MAGIC Software Development, 2000-2007
+#   Copyright: MAGIC Software Development, 2000-2008
 #
 #
@@ -32 +32 @@
 source `dirname $0`/sourcefile
 printprocesslog "INFO starting $0"
-echo "This script has not been adapted to the new file structure (MAGIC II). "
-echo "Please adapt it before using it."
-exit
-
 program=compmux
 column=fCompmux
@@ -41 +37 @@
 set -C
 
-scriptlog=/magic/datacenter/compmux/$program-$datetime.log
+scriptlog=$runlogpath/run$program-$datetime.log
 date >> $scriptlog 2>&1
-todofile=$listpath/ToDo-$table-$column.txt
 
-if [ -e $todofile ]
-then 
-   echo "$program is already running -> exit" >> $scriptlog 2>&1
-   exit
-fi
-
-#get todo list
-getdolist >> $scriptlog 2>&1
-
-#retrieving runs from todo file
-files=(`cat $listpath/ToDo-$table-$column.txt`)
-if [ "$files" = "" ]
-then 
-   echo "nothing to do -> exit" >> $scriptlog 2>&1
-   finish >> $scriptlog 2>&1
-fi
-
-for file in ${files[@]}
+# query db 20 times for jobs
+for ((j=0 ; j < 20 ; j++))
 do
-   echo -e "\ngetting path for run $file..." >> $scriptlog 2>&1
-   fullpath=`find $datapath/rawfiles/2007/0[2-9] -name *${file}_[DCPS]_*`
+   gettodo "1" >> $scriptlog 2>&1
+   runno=${primaries[0]}
+   tempfile=/data/tmp/temp${runno}.gz
    
-   lockfile=$lockpath/lock-$program-$file.txt
+   lockfile=$lockpath/lock-$table-$column-$runno.txt
+   checklock >> $scriptlog 2>&1
    
-   #check lock file and stop for this run if already being processed
-   checklock continue >> $scriptlog 2>&1
-   
-   primvar=$file
    setstatus "start" >> $scriptlog 2>&1
    
-   cd $mars
+   # try to find the file; if impossible, continue with an error
+   echo "getting path for run $runno..." >> $scriptlog 2>&1
+   printprocesslog "INFO starting $program for run $runno"
+   fullname=`find $datapath/rawfiles/2007/0[2-9] -name 20070*${runno}_[DCPS]_*.raw.gz`
+   arraycheck=`echo $fullname | wc -w`
+   if [ "$fullname" == "" ] || ! [ $arraycheck == 1 ]
+   then
+      echo "ERROR file for run $runno not found or more than one file found... continue" >> $scriptlog 2>&1
+      printprocesslog "ERROR $program failed, file for run $runno not found or more than one file found"
+      rm -v $lockpath/lock-$table-$column-$runno.txt >> $scriptlog 2>&1
+      check=20
+      com=$FCompmux
+      setstatus "stop" >> $scriptlog 2>&1
+      echo "" >> $scriptlog 2>&1
+      continue
+   fi
    
-   #cut the slices
-   gzip -cd $fullpath | $mars/datacenter/compmux | gzip -1c > /tmp/temp${file}.gz
-   check1=$?
-   touch -r $fullpath /tmp/temp${file}.gz
+   # cut the slices
+   check1=`ssh -nx phoenix 'cd /home/operator/compmux ; nice -n 19 gzip -cd' $fullname '| nice -n 19 ./compmux | nice -n 19 gzip -1c > /data/tmp/temp'${runno}.gz ' ; echo ${PIPESTATUS[1]}'`
    
-   #check success and insert information into the database
+   # check success and insert information into the database
    case $check1 in
-      0)   echo " check1=$check1 -> everything ok -> compmux succeeded, overwriting old file for run $file" >> $scriptlog 2>&1
-           mv -f /tmp/temp${file}.gz $fullpath >> $scriptlog 2>&1
+      0)   echo " check1=$check1 -> everything ok -> compmux succeeded, overwriting old file for run $runno" >> $scriptlog 2>&1
+           ssh -nx phoenix nice -n 19 touch -r $fullname $tempfile
+           ssh -nx phoenix nice -n 19 mv -f $tempfile $fullname >> $scriptlog 2>&1
            check2=$?
-           chmod a=r $fullpath
            case $check2 in
-              0)   echo " check2=$check2 -> everything ok -> mv done, setting db status for run $file" >> $scriptlog 2>&1
-                   rm -v $lockfile >> $scriptlog 2>&1
+              0)   echo " check2=$check2 -> everything ok -> mv done, setting db status for run $runno" >> $scriptlog 2>&1
+                   ssh -nx phoenix nice -n 19 chmod a=r $fullname
                    ;;
-              *)   echo " check2=$check2 -> ERROR - moving of run $file failed -> continue" >> $scriptlog 2>&1
-                   continue
+              *)   echo " check2=$check2 -> ERROR - moving of run $runno failed" >> $scriptlog 2>&1
+                   printprocesslog "ERROR $program failed, moving of run $runno failed"
+                   check=$check2
+                   com=$FCompmux
                    ;;
            esac
+           printprocesslog "INFO $program finished successfully for run $runno"
            ;;
-      *)   echo " check1=$check1 -> ERROR - $program failed for run $file" >> $scriptlog 2>&1
-           com=$FCompmux
+      *)   echo " check1=$check1 -> ERROR - $program failed for run $runno" >> $scriptlog 2>&1
+           printprocesslog "ERROR $program failed for run $runno"
+           rm -v $tempfile >> $scriptlog 2>&1
            check=$check1
+           com=$FCompmux
            ;;
    esac
    
+   rm -v $lockpath/lock-$table-$column-$runno.txt >> $scriptlog 2>&1
    setstatus "stop" >> $scriptlog 2>&1
+   echo "" >> $scriptlog 2>&1
 done
+
+finish >> $scriptlog 2>&1

trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles

@@ -21 +21 @@
 #   Author(s): Stefan Ruegamer 02/2007 <mailto:snruegam@astro.uni-wuerzburg.de>
 #
-#   Copyright: MAGIC Software Development, 2000-2007
+#   Copyright: MAGIC Software Development, 2000-2008
 #
 #
@@ -50 +50 @@
 # change permission for files
 ssh tape@dc07 chmod -R g+w /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1
-# output for chmod with -v or -c (only changes)
-
-# outdated, .finished files are no longer used
-#printprocesslog "INFO moving directories from fromlapalma to fromtape"
-# find directories which have to be copied
-#dirs=`ls /magic/datacenter/fromlapalma/RAWchk/*/*.finished | sed -e 's/RAWchk/RAW/g' | cut -d/ -f1-6`
-#if [ ! "$dirs" == "" ]
-#then
-   # change permission for this directories
-   #ssh lapalma@dc09 chmod -R g+w $dirs >> $scriptlog 2>&1
-   # move directories to the tapedirectory
-   #mv -v $dirs /magic/datacenter/fromtape/rawdata/ >> $scriptlog 2>&1
-#fi
 
 printprocesslog "INFO moving directories from muxdata to rawdata"
@@ -73 +60 @@
 fi
 
-
 # find rawfiles
-rawfiles=`find /magic/datacenter/fromtape/rawdata/ -name '*.*'`
+rawfiles=`find /magic/{datacenter,data/rawfiles}/fromtape/*data/ -regextype posix-egrep -regex '.*/20[01][0-9]{5}_(M[12]_)?[0-9]{8}(\.[0-9]{3})?_.*.raw' -type f`
 
 if [ "$rawfiles" == "" ]
@@ -88 +74 @@
 # workaround for rawfiles with wrong timing
 #   newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles-wrong-timing/g' -e 's/_/\//1' -e 's/_/\//1'`
-   newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'`
+   path=`dirname $rawfile | cut -d/ -f3`
+   if [ "$path" == "datacenter" ]
+   then
+      newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'`
+   elif [ "$path" == "data" ]
+   then
+      newrawfile=`echo $rawfile | sed -e 's/\/fromtape\/muxdata//g' -e 's/_/\//1' -e 's/_/\//1'`
+   else
+      echo "ERROR something is wrong with the path of file $rawfile -> continue" >> $scriptlog 2>&1
+      printprocesslog "ERROR moving rawfile $rawfile failed"
+      continue
+   fi
+   
    newdir=`dirname $newrawfile`
    makedir $newdir >> $scriptlog 2>&1
@@ -95 +93 @@
 done
 
-rmdir -v /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1
+rmdir -v /magic/{datacenter,data/rawfiles}/fromtape/*data/* >> $scriptlog 2>&1
 
 printprocesslog "INFO launching filesondisk"
 echo "launching filesondisk" >> $scriptlog 2>&1
-$scriptspath/filesondisk&
+$scriptspath/filesondisk &
 
 finish >> $scriptlog 2>&1
-

trunk/MagicSoft/Mars/datacenter/scripts/setup

@@ -20 +20 @@
 #   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
 #
-#   Copyright: MAGIC Software Development, 2000-2008
+#   Copyright: MAGIC Software Development, 2000-2007
 #
 #
@@ -66 +66 @@
 sleeptimelimit=360 #360
 errorsleeptimedefault=60 #60
-#max=18 #maximum number of processes
 max=42 #maximum number of processes
 
@@ -73 +72 @@
 #datascripts=( "runcallisto" "runstar" "runganymed" )
 #datascriptscolname=( "fCallisto" "fStar" "fGanymed" )
-datascripts=( "runganymed" "runstar" "runcallisto" )
-datascriptscolname=( "fGanymed" "fStar" "fCallisto" )
+datascripts=( "runganymed" "runstar" "runcallisto" "cutslices" )
+datascriptscolname=( "fGanymed" "fStar" "fCallisto" "fCompmux" )
 
 #
@@ -91 +90 @@
 pnostar=(        10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 )
 pnoganymed=(      7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7 )
+pnocutslices=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
 #pnodatacheck=(   20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
 
@@ -96 +96 @@
 pnostarwe=(      10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 )
 pnoganymedwe=(    7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7 )
+pnocutsliceswe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
 #pnodatacheckwe=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
 
-#pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ${pnodatacheck[@]} )
-#pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ${pnodatacheckwe[@]} )
+##pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ${pnodatacheck[@]} )
+##pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ${pnodatacheckwe[@]} )
 #pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} )
 #pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} )
-pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} )
-pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} )
+pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ${pnocutslices[@]} )
+pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ${pnocutsliceswe[@]} )
 
+# values for mc production
+mcscripts=( "runcorsika" "runreflector" "runcamera" )
+mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
 
-# not yet working completely
+#hour:            0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 
-## values for mc production
-#mcscripts=( "runcorsika" "runreflector" "runcamera" )
-#mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
-#
-##hour:            0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
-#
-#pnocorsika=(     12 12 12 12 12 12 12 10  9  8  7  7  7  7  7  7  7  7  7  8  9 10 12 12 )
-#pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
-#pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
-#
-#pnocorsikawe=(   12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 )
-#pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
-#pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
-#
+pnocorsika=(     18 18 18 18 18 18 18 12 10  8  7  7  7  7  7  7  7  7  7  8 10 12 18 18 )
+pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
+pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
+
+pnocorsikawe=(   18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 )
+pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
+pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
+
 #pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} )
 #pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )