Changeset 9081 for trunk/MagicSoft


Ignore:
Timestamp:
08/04/08 13:41:17 (16 years ago)
Author:
snruegam
Message:
*** empty log message ***
Location:
trunk/MagicSoft/Mars
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/MagicSoft/Mars/Changelog

    r9080 r9081  
    1818
    1919                                                 -*-*- END OF LINE -*-*-
     20
     21 2008/08/04 Stefan Ruegamer
     22
     23   * datacenter/scripts/setup:
     24     - added compux/cutslices to the jobmanager queue
     25 
     26 * datacenter/scripts/movingrawfiles:
     27     - modified the regex for finding the run number such that it
     28       satisfies the new raw file convention
     29     - the script now searches for files in /magic and /data, so
     30       movingrawfiles2 is obsolete
     31
     32 * datacenter/scripts/cutslices:
     33     - adapted the script for jobmanager use
     34     - included some more checks
     35
     36
    2037
    2138 2008/08/04 Thomas Bretz

  • trunk/MagicSoft/Mars/datacenter/scripts/cutslices

    r9046 r9081  
    2020#   Author(s): Stefan Ruegamer 12/2007 <mailto:snruegam@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2007
     22#   Copyright: MAGIC Software Development, 2000-2008
    2323#
    2424#
    … …  
    3232source `dirname $0`/sourcefile
    3333printprocesslog "INFO starting $0"
    34 echo "This script has not been adapted to the new file structure (MAGIC II). "
    35 echo "Please adapt it before using it."
    36 exit
    37 
    3834program=compmux
    3935column=fCompmux
    … …  
    4137set -C
    4238
    43 scriptlog=/magic/datacenter/compmux/$program-$datetime.log
     39scriptlog=$runlogpath/run$program-$datetime.log
    4440date >> $scriptlog 2>&1
    45 todofile=$listpath/ToDo-$table-$column.txt
    4641
    47 if [ -e $todofile ]
    48 then
    49    echo "$program is already running -> exit" >> $scriptlog 2>&1
    50    exit
    51 fi
    52 
    53 #get todo list
    54 getdolist >> $scriptlog 2>&1
    55 
    56 #retrieving runs from todo file
    57 files=(`cat $listpath/ToDo-$table-$column.txt`)
    58 if [ "$files" = "" ]
    59 then
    60    echo "nothing to do -> exit" >> $scriptlog 2>&1
    61    finish >> $scriptlog 2>&1
    62 fi
    63 
    64 for file in ${files[@]}
     42# query db 20 times for jobs
     43for ((j=0 ; j < 20 ; j++))
    6544do
    66    echo -e "\ngetting path for run $file..." >> $scriptlog 2>&1
    67    fullpath=`find $datapath/rawfiles/2007/0[2-9] -name *${file}_[DCPS]_*`
     45   gettodo "1" >> $scriptlog 2>&1
     46   runno=${primaries[0]}
     47   tempfile=/data/tmp/temp${runno}.gz
    6848   
    69    lockfile=$lockpath/lock-$program-$file.txt
     49   lockfile=$lockpath/lock-$table-$column-$runno.txt
     50   checklock >> $scriptlog 2>&1
    7051   
    71    #check lock file and stop for this run if already being processed
    72    checklock continue >> $scriptlog 2>&1
    73    
    74    primvar=$file
    7552   setstatus "start" >> $scriptlog 2>&1
    7653   
    77    cd $mars
     54   # try to find the file; if impossible, continue with an error
     55   echo "getting path for run $runno..." >> $scriptlog 2>&1
     56   printprocesslog "INFO starting $program for run $runno"
     57   fullname=`find $datapath/rawfiles/2007/0[2-9] -name 20070*${runno}_[DCPS]_*.raw.gz`
     58   arraycheck=`echo $fullname | wc -w`
     59   if [ "$fullname" == "" ] || ! [ $arraycheck == 1 ]
     60   then
     61      echo "ERROR file for run $runno not found or more than one file found... continue" >> $scriptlog 2>&1
     62      printprocesslog "ERROR $program failed, file for run $runno not found or more than one file found"
     63      rm -v $lockpath/lock-$table-$column-$runno.txt >> $scriptlog 2>&1
     64      check=20
     65      com=$FCompmux
     66      setstatus "stop" >> $scriptlog 2>&1
     67      echo "" >> $scriptlog 2>&1
     68      continue
     69   fi
    7870   
    79    #cut the slices
    80    gzip -cd $fullpath | $mars/datacenter/compmux | gzip -1c > /tmp/temp${file}.gz
    81    check1=$?
    82    touch -r $fullpath /tmp/temp${file}.gz
     71   # cut the slices
     72   check1=`ssh -nx phoenix 'cd /home/operator/compmux ; nice -n 19 gzip -cd' $fullname '| nice -n 19 ./compmux | nice -n 19 gzip -1c > /data/tmp/temp'${runno}.gz ' ; echo ${PIPESTATUS[1]}'`
    8373   
    84    #check success and insert information into the database
     74   # check success and insert information into the database
    8575   case $check1 in
    86       0)   echo " check1=$check1 -> everything ok -> compmux succeeded, overwriting old file for run $file" >> $scriptlog 2>&1
    87            mv -f /tmp/temp${file}.gz $fullpath >> $scriptlog 2>&1
     76      0)   echo " check1=$check1 -> everything ok -> compmux succeeded, overwriting old file for run $runno" >> $scriptlog 2>&1
     77           ssh -nx phoenix nice -n 19 touch -r $fullname $tempfile
     78           ssh -nx phoenix nice -n 19 mv -f $tempfile $fullname >> $scriptlog 2>&1
    8879           check2=$?
    89            chmod a=r $fullpath
    9080           case $check2 in
    91               0)   echo " check2=$check2 -> everything ok -> mv done, setting db status for run $file" >> $scriptlog 2>&1
    92                    rm -v $lockfile >> $scriptlog 2>&1
     81              0)   echo " check2=$check2 -> everything ok -> mv done, setting db status for run $runno" >> $scriptlog 2>&1
     82                   ssh -nx phoenix nice -n 19 chmod a=r $fullname
    9383                   ;;
    94               *)   echo " check2=$check2 -> ERROR - moving of run $file failed -> continue" >> $scriptlog 2>&1
    95                    continue
     84              *)   echo " check2=$check2 -> ERROR - moving of run $runno failed" >> $scriptlog 2>&1
     85                   printprocesslog "ERROR $program failed, moving of run $runno failed"
     86                   check=$check2
     87                   com=$FCompmux
    9688                   ;;
    9789           esac
     90           printprocesslog "INFO $program finished successfully for run $runno"
    9891           ;;
    99       *)   echo " check1=$check1 -> ERROR - $program failed for run $file" >> $scriptlog 2>&1
    100            com=$FCompmux
     92      *)   echo " check1=$check1 -> ERROR - $program failed for run $runno" >> $scriptlog 2>&1
     93           printprocesslog "ERROR $program failed for run $runno"
     94           rm -v $tempfile >> $scriptlog 2>&1
    10195           check=$check1
     96           com=$FCompmux
    10297           ;;
    10398   esac
    10499   
     100   rm -v $lockpath/lock-$table-$column-$runno.txt >> $scriptlog 2>&1
    105101   setstatus "stop" >> $scriptlog 2>&1
     102   echo "" >> $scriptlog 2>&1
    106103done
     104
     105finish >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles

    r8682 r9081  
    2121#   Author(s): Stefan Ruegamer 02/2007 <mailto:snruegam@astro.uni-wuerzburg.de>
    2222#
    23 #   Copyright: MAGIC Software Development, 2000-2007
     23#   Copyright: MAGIC Software Development, 2000-2008
    2424#
    2525#
    … …  
    5050# change permission for files
    5151ssh tape@dc07 chmod -R g+w /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1
    52 # output for chmod with -v or -c (only changes)
    53 
    54 # outdated, .finished files are no longer used
    55 #printprocesslog "INFO moving directories from fromlapalma to fromtape"
    56 # find directories which have to be copied
    57 #dirs=`ls /magic/datacenter/fromlapalma/RAWchk/*/*.finished | sed -e 's/RAWchk/RAW/g' | cut -d/ -f1-6`
    58 #if [ ! "$dirs" == "" ]
    59 #then
    60    # change permission for this directories
    61    #ssh lapalma@dc09 chmod -R g+w $dirs >> $scriptlog 2>&1
    62    # move directories to the tapedirectory
    63    #mv -v $dirs /magic/datacenter/fromtape/rawdata/ >> $scriptlog 2>&1
    64 #fi
    6552
    6653printprocesslog "INFO moving directories from muxdata to rawdata"
    … …  
    7360fi
    7461
    75 
    7662# find rawfiles
    77 rawfiles=`find /magic/datacenter/fromtape/rawdata/ -name '*.*'`
     63rawfiles=`find /magic/{datacenter,data/rawfiles}/fromtape/*data/ -regextype posix-egrep -regex '.*/20[01][0-9]{5}_(M[12]_)?[0-9]{8}(\.[0-9]{3})?_.*.raw' -type f`
    7864
    7965if [ "$rawfiles" == "" ]
    … …  
    8874# workaround for rawfiles with wrong timing
    8975#   newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles-wrong-timing/g' -e 's/_/\//1' -e 's/_/\//1'`
    90    newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'`
     76   path=`dirname $rawfile | cut -d/ -f3`
     77   if [ "$path" == "datacenter" ]
     78   then
     79      newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'`
     80   elif [ "$path" == "data" ]
     81   then
     82      newrawfile=`echo $rawfile | sed -e 's/\/fromtape\/muxdata//g' -e 's/_/\//1' -e 's/_/\//1'`
     83   else
     84      echo "ERROR something is wrong with the path of file $rawfile -> continue" >> $scriptlog 2>&1
     85      printprocesslog "ERROR moving rawfile $rawfile failed"
     86      continue
     87   fi
     88   
    9189   newdir=`dirname $newrawfile`
    9290   makedir $newdir >> $scriptlog 2>&1
    … …  
    9593done
    9694
    97 rmdir -v /magic/datacenter/fromtape/rawdata/* >> $scriptlog 2>&1
     95rmdir -v /magic/{datacenter,data/rawfiles}/fromtape/*data/* >> $scriptlog 2>&1
    9896
    9997printprocesslog "INFO launching filesondisk"
    10098echo "launching filesondisk" >> $scriptlog 2>&1
    101 $scriptspath/filesondisk&
     99$scriptspath/filesondisk &
    102100
    103101finish >> $scriptlog 2>&1
    104 

  • trunk/MagicSoft/Mars/datacenter/scripts/setup

    r9076 r9081  
    2020#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2008
     22#   Copyright: MAGIC Software Development, 2000-2007
    2323#
    2424#
    … …  
    6666sleeptimelimit=360 #360
    6767errorsleeptimedefault=60 #60
    68 #max=18 #maximum number of processes
    6968max=42 #maximum number of processes
    7069
    … …  
    7372#datascripts=( "runcallisto" "runstar" "runganymed" )
    7473#datascriptscolname=( "fCallisto" "fStar" "fGanymed" )
    75 datascripts=( "runganymed" "runstar" "runcallisto" )
    76 datascriptscolname=( "fGanymed" "fStar" "fCallisto" )
     74datascripts=( "runganymed" "runstar" "runcallisto" "cutslices" )
     75datascriptscolname=( "fGanymed" "fStar" "fCallisto" "fCompmux" )
    7776
    7877#
    … …  
    9190pnostar=(        10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 )
    9291pnoganymed=(      7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7 )
     92pnocutslices=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    9393#pnodatacheck=(   20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
    9494
    … …  
    9696pnostarwe=(      10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 )
    9797pnoganymedwe=(    7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7  7 )
     98pnocutsliceswe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    9899#pnodatacheckwe=( 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 )
    99100
    100 #pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ${pnodatacheck[@]} )
    101 #pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ${pnodatacheckwe[@]} )
     101##pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} ${pnodatacheck[@]} )
     102##pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} ${pnodatacheckwe[@]} )
    102103#pnosweek=( ${pnocallisto[@]} ${pnostar[@]} ${pnoganymed[@]} )
    103104#pnoswe=( ${pnocallistowe[@]} ${pnostarwe[@]} ${pnoganymedwe[@]} )
    104 pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} )
    105 pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} )
     105pnosweek=( ${pnoganymed[@]} ${pnostar[@]} ${pnocallisto[@]} ${pnocutslices[@]} )
     106pnoswe=( ${pnoganymedwe[@]} ${pnostarwe[@]} ${pnocallistowe[@]} ${pnocutsliceswe[@]} )
    106107
     108# values for mc production
     109mcscripts=( "runcorsika" "runreflector" "runcamera" )
     110mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
    107111
    108 # not yet working completely
     112#hour:            0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
    109113
    110 ## values for mc production
    111 #mcscripts=( "runcorsika" "runreflector" "runcamera" )
    112 #mcscriptscolname=( "fCorsikaFileAvail" "fReflectorFileAvail" "fCameraFileAvail" )
    113 #
    114 ##hour:            0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
    115 #
    116 #pnocorsika=(     12 12 12 12 12 12 12 10  9  8  7  7  7  7  7  7  7  7  7  8  9 10 12 12 )
    117 #pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    118 #pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
    119 #
    120 #pnocorsikawe=(   12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 )
    121 #pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
    122 #pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
    123 #
     114pnocorsika=(     18 18 18 18 18 18 18 12 10  8  7  7  7  7  7  7  7  7  7  8 10 12 18 18 )
     115pnoreflector=(    8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
     116pnocamera=(       4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
     117
     118pnocorsikawe=(   18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 18 )
     119pnoreflectorwe=(  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8  8 )
     120pnocamerawe=(     4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4  4 )
     121
    124122#pnosweek=( ${pnocorsika[@]} ${pnoreflector[@]} ${pnocamera[@]} )
    125123#pnoswe=( ${pnocorsikawe[@]} ${pnoreflectorwe[@]} ${pnocamerawe[@]} )
Note: See TracChangeset for help on using the changeset viewer.