Index: trunk/MagicSoft/Mars/datacenter/scripts/cutslices =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/cutslices (revision 9076) +++ trunk/MagicSoft/Mars/datacenter/scripts/cutslices (revision 9081) @@ -20,5 +20,5 @@ # Author(s): Stefan Ruegamer 12/2007 # -# Copyright: MAGIC Software Development, 2000-2007 +# Copyright: MAGIC Software Development, 2000-2008 # # @@ -32,8 +32,4 @@ source `dirname $0`/sourcefile printprocesslog "INFO starting $0" -echo "This script has not been adapted to the new file structure (MAGIC II). " -echo "Please adapt it before using it." -exit - program=compmux column=fCompmux @@ -41,66 +37,69 @@ set -C -scriptlog=/magic/datacenter/compmux/$program-$datetime.log +scriptlog=$runlogpath/run$program-$datetime.log date >> $scriptlog 2>&1 -todofile=$listpath/ToDo-$table-$column.txt -if [ -e $todofile ] -then - echo "$program is already running -> exit" >> $scriptlog 2>&1 - exit -fi - -#get todo list -getdolist >> $scriptlog 2>&1 - -#retrieving runs from todo file -files=(`cat $listpath/ToDo-$table-$column.txt`) -if [ "$files" = "" ] -then - echo "nothing to do -> exit" >> $scriptlog 2>&1 - finish >> $scriptlog 2>&1 -fi - -for file in ${files[@]} +# query db 20 times for jobs +for ((j=0 ; j < 20 ; j++)) do - echo -e "\ngetting path for run $file..." >> $scriptlog 2>&1 - fullpath=`find $datapath/rawfiles/2007/0[2-9] -name *${file}_[DCPS]_*` + gettodo "1" >> $scriptlog 2>&1 + runno=${primaries[0]} + tempfile=/data/tmp/temp${runno}.gz - lockfile=$lockpath/lock-$program-$file.txt + lockfile=$lockpath/lock-$table-$column-$runno.txt + checklock >> $scriptlog 2>&1 - #check lock file and stop for this run if already being processed - checklock continue >> $scriptlog 2>&1 - - primvar=$file setstatus "start" >> $scriptlog 2>&1 - cd $mars + # try to find the file; if impossible, continue with an error + echo "getting path for run $runno..." >> $scriptlog 2>&1 + printprocesslog "INFO starting $program for run $runno" + fullname=`find $datapath/rawfiles/2007/0[2-9] -name 20070*${runno}_[DCPS]_*.raw.gz` + arraycheck=`echo $fullname | wc -w` + if [ "$fullname" == "" ] || ! [ $arraycheck == 1 ] + then + echo "ERROR file for run $runno not found or more than one file found... continue" >> $scriptlog 2>&1 + printprocesslog "ERROR $program failed, file for run $runno not found or more than one file found" + rm -v $lockpath/lock-$table-$column-$runno.txt >> $scriptlog 2>&1 + check=20 + com=$FCompmux + setstatus "stop" >> $scriptlog 2>&1 + echo "" >> $scriptlog 2>&1 + continue + fi - #cut the slices - gzip -cd $fullpath | $mars/datacenter/compmux | gzip -1c > /tmp/temp${file}.gz - check1=$? - touch -r $fullpath /tmp/temp${file}.gz + # cut the slices + check1=`ssh -nx phoenix 'cd /home/operator/compmux ; nice -n 19 gzip -cd' $fullname '| nice -n 19 ./compmux | nice -n 19 gzip -1c > /data/tmp/temp'${runno}.gz ' ; echo ${PIPESTATUS[1]}'` - #check success and insert information into the database + # check success and insert information into the database case $check1 in - 0) echo " check1=$check1 -> everything ok -> compmux succeeded, overwriting old file for run $file" >> $scriptlog 2>&1 - mv -f /tmp/temp${file}.gz $fullpath >> $scriptlog 2>&1 + 0) echo " check1=$check1 -> everything ok -> compmux succeeded, overwriting old file for run $runno" >> $scriptlog 2>&1 + ssh -nx phoenix nice -n 19 touch -r $fullname $tempfile + ssh -nx phoenix nice -n 19 mv -f $tempfile $fullname >> $scriptlog 2>&1 check2=$? - chmod a=r $fullpath case $check2 in - 0) echo " check2=$check2 -> everything ok -> mv done, setting db status for run $file" >> $scriptlog 2>&1 - rm -v $lockfile >> $scriptlog 2>&1 + 0) echo " check2=$check2 -> everything ok -> mv done, setting db status for run $runno" >> $scriptlog 2>&1 + ssh -nx phoenix nice -n 19 chmod a=r $fullname ;; - *) echo " check2=$check2 -> ERROR - moving of run $file failed -> continue" >> $scriptlog 2>&1 - continue + *) echo " check2=$check2 -> ERROR - moving of run $runno failed" >> $scriptlog 2>&1 + printprocesslog "ERROR $program failed, moving of run $runno failed" + check=$check2 + com=$FCompmux ;; esac + printprocesslog "INFO $program finished successfully for run $runno" ;; - *) echo " check1=$check1 -> ERROR - $program failed for run $file" >> $scriptlog 2>&1 - com=$FCompmux + *) echo " check1=$check1 -> ERROR - $program failed for run $runno" >> $scriptlog 2>&1 + printprocesslog "ERROR $program failed for run $runno" + rm -v $tempfile >> $scriptlog 2>&1 check=$check1 + com=$FCompmux ;; esac + rm -v $lockpath/lock-$table-$column-$runno.txt >> $scriptlog 2>&1 setstatus "stop" >> $scriptlog 2>&1 + echo "" >> $scriptlog 2>&1 done + +finish >> $scriptlog 2>&1 Index: trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles (revision 9076) +++ trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles (revision 9081) @@ -21,5 +21,5 @@ # Author(s): Stefan Ruegamer 02/2007 # -# Copyright: MAGIC Software Development, 2000-2007 +# Copyright: MAGIC Software Development, 2000-2008 # # @@ -50,17 +50,4 @@ # change permission for files ssh tape@dc07 chmod -R g+w /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1 -# output for chmod with -v or -c (only changes) - -# outdated, .finished files are no longer used -#printprocesslog "INFO moving directories from fromlapalma to fromtape" -# find directories which have to be copied -#dirs=`ls /magic/datacenter/fromlapalma/RAWchk/*/*.finished | sed -e 's/RAWchk/RAW/g' | cut -d/ -f1-6` -#if [ ! "$dirs" == "" ] -#then - # change permission for this directories - #ssh lapalma@dc09 chmod -R g+w $dirs >> $scriptlog 2>&1 - # move directories to the tapedirectory - #mv -v $dirs /magic/datacenter/fromtape/rawdata/ >> $scriptlog 2>&1 -#fi printprocesslog "INFO moving directories from muxdata to rawdata" @@ -73,7 +60,6 @@ fi - # find rawfiles -rawfiles=`find /magic/datacenter/fromtape/rawdata/ -name '*.*'` +rawfiles=`find /magic/{datacenter,data/rawfiles}/fromtape/*data/ -regextype posix-egrep -regex '.*/20[01][0-9]{5}_(M[12]_)?[0-9]{8}(\.[0-9]{3})?_.*.raw' -type f` if [ "$rawfiles" == "" ] @@ -88,5 +74,17 @@ # workaround for rawfiles with wrong timing # newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles-wrong-timing/g' -e 's/_/\//1' -e 's/_/\//1'` - newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'` + path=`dirname $rawfile | cut -d/ -f3` + if [ "$path" == "datacenter" ] + then + newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'` + elif [ "$path" == "data" ] + then + newrawfile=`echo $rawfile | sed -e 's/\/fromtape\/muxdata//g' -e 's/_/\//1' -e 's/_/\//1'` + else + echo "ERROR something is wrong with the path of file $rawfile -> continue" >> $scriptlog 2>&1 + printprocesslog "ERROR moving rawfile $rawfile failed" + continue + fi + newdir=`dirname $newrawfile` makedir $newdir >> $scriptlog 2>&1 @@ -95,10 +93,9 @@ done -rmdir -v /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1 +rmdir -v /magic/{datacenter,data/rawfiles}/fromtape/*data/* >> $scriptlog 2>&1 printprocesslog "INFO launching filesondisk" echo "launching filesondisk" >> $scriptlog 2>&1 -$scriptspath/filesondisk& +$scriptspath/filesondisk & finish >> $scriptlog 2>&1 - Index: trunk/MagicSoft/Mars/datacenter/scripts/setup =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/setup (revision 9076) +++ trunk/MagicSoft/Mars/datacenter/scripts/setup (revision 9081) @@ -20,5 +20,5 @@ # Author(s): Daniela Dorner 05/2005 # -# Copyright: MAGIC Software Development, 2000-2008 +# Copyright: MAGIC Software Development, 2000-2007 # # @@ -66,5 +66,4 @@ sleeptimelimit=360 #360 errorsleeptimedefault=60 #60 -#max=18 #maximum number of processes max=42 #maximum number of processes @@ -73,6 +72,6 @@ #datascripts=( "runcallisto" "runstar" "runganymed" ) #datascriptscolname=( "fCallisto" "fStar" "fGanymed" ) -datascripts=( "runganymed" "runstar" "runcallisto" ) -datascriptscolname=( "fGanymed" "fStar" "fCallisto" ) +datascripts=( "runganymed" "runstar" "runcallisto" "cutslices" ) +datascriptscolname=( "fGanymed" "fStar" "fCallisto" "fCompmux" ) # @@ -91,4 +90,5 @@ pnostar=( 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 ) pnoganymed=( 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ) +pnocutslices=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) #pnodatacheck=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 ) @@ -96,30 +96,28 @@ pnostarwe=( 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 ) pnoganymedwe=( 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ) +pnocutsliceswe=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) #pnodatacheckwe=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 ) -#pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ${pnodatacheck[@]} ) -#pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ${pnodatacheckwe[@]} ) +##pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ${pnodatacheck[@]} ) +##pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ${pnodatacheckwe[@]} ) #pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ) #pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ) -pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ) -pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ) +pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ${pnocutslices[@]} ) +pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ${pnocutsliceswe[@]} ) +# values for mc production +mcscripts=( "runcorsika" "runreflector" "runcamera" ) +mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" ) -# not yet working completely +#hour: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 -## values for mc production -#mcscripts=( "runcorsika" "runreflector" "runcamera" ) -#mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" ) -# -##hour: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 -# -#pnocorsika=( 12 12 12 12 12 12 12 10 9 8 7 7 7 7 7 7 7 7 7 8 9 10 12 12 ) -#pnoreflector=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) -#pnocamera=( 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ) -# -#pnocorsikawe=( 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 ) -#pnoreflectorwe=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) -#pnocamerawe=( 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ) -# +pnocorsika=( 18 18 18 18 18 18 18 12 10 8 7 7 7 7 7 7 7 7 7 8 10 12 18 18 ) +pnoreflector=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) +pnocamera=( 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ) + +pnocorsikawe=( 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 ) +pnoreflectorwe=( 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 ) +pnocamerawe=( 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ) + #pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} ) #pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )