Ignore:
Timestamp:
01/10/06 22:12:58 (19 years ago)
Author:
Daniela Dorner
Message:
*** empty log message ***
Location:
trunk/MagicSoft/Mars/datacenter/scripts
Files:
27 edited

Legend:

Unmodified
Added
Removed

  • trunk/MagicSoft/Mars/datacenter/scripts/buildsequenceentries

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the building of sequences for all days for which
     28# this step is needed.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then the sequences are built for each date in the todolist using the
     33# macro buildsequenceentries.C
     34# If this was successful, the status is inserted into the database using
     35# the macro setstatus.C
    2736#
    2837
    … …  
    5261makedir $getstatuslogpath >> $scriptlog 2>&1
    5362
     63# check if script is already running
    5464date > $lockpath/lock-buildsequenceentries.txt >> $scriptlog 2>&1
    5565checklock0=$?
    … …  
    5767    0)   echo "checklock0=$checklock0 -> continue " >> $scriptlog 2>&1;;
    5868    1)   echo "checklock0=$checklock0 -> file exists " >> $scriptlog 2>&1
    59          echo "-> getting list for buildsequenceentries is running -> exit" >> $scriptlog 2>&1
     69         echo "-> buildsequenceentries is running -> exit" >> $scriptlog 2>&1
    6070         date  >> $scriptlog 2>&1
    6171         exit;;
    … …  
    6373esac
    6474
    65 
    66 echo "getting list..." >> $scriptlog 2>&1
     75# get todo list
     76echo "getting todo list..." >> $scriptlog 2>&1
    6777check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatus.log | grep int | sed -e 's/(int)//'`
    6878
    6979case $check0 in
    70    1)   echo "check0=$check0 -> everything ok -> do step" >> $scriptlog 2>&1;;
    71    *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     80   1)   echo "check0=$check0 -> everything ok, got todo list -> build sequence entries" >> $scriptlog 2>&1;;
     81   *)   echo "check0=$check0 -> ERROR -> could not get todo list -> exit" >> $scriptlog 2>&1
     82        rm -v $todofile >> $scriptlog 2>&1
     83        rm -v $lockpath/lock-buildsequenceentries.txt >> $scriptlog 2>&1
     84        date  >> $scriptlog 2>&1
     85        exit;;
    7286esac
    7387
     88# retrieve dates from todofile
    7489dates=(`cat $todofile`)
    75 
    7690if [ "$dates" = "" ]
    7791then
    … …  
    8498
    8599echo "dates: "${dates[@]} >> $scriptlog 2>&1
    86 
    87100for date in ${dates[@]}
    88101do
    … …  
    95108
    96109   case $check1 in
    97       1)   echo "check1=$check1 -> everything ok -> setting status..." >> $scriptlog 2>&1
     110      1)   echo "check1=$check1 -> everything ok -> setting status for $date..." >> $scriptlog 2>&1
    98111           setstatuslogpath=$logpath/setstatus/buildsequenceentries/$year2
    99112           makedir $setstatuslogpath >> $scriptlog 2>&1
    … …  
    102115           case $check2 in
    103116              1)   echo "check2=$check2 -> everything ok, status has been set" >> $scriptlog 2>&1;;
    104               *)   echo "check2=$check2 -> ERROR -> step could not be set" >> $scriptlog 2>&1;;
     117              *)   echo "check2=$check2 -> ERROR -> step could not be set for $date" >> $scriptlog 2>&1;;
    105118           esac
    106119           ;;
    107       *)   echo "check1=$check1 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     120      *)   echo "check1=$check1 -> ERROR - buildsequenceentries.C failed for $date -> step has to be repeated" >> $scriptlog 2>&1;;
    108121   esac
    109122done

  • trunk/MagicSoft/Mars/datacenter/scripts/checkfilesforsequenceavail

    r7426 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the checking of the availability of the files for
     28# the sequences.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each sequence the availability of the files (raw, cc, caco) is
     33# checked by using the macro checkfilesavail.C
     34# If this was successful, the status is inserted into the database using
     35# the macro setstatus.C
    2736#
    2837
    … …  
    5160cd $mars
    5261
    53 if [ -e $todofile ]
    54 then
    55    echo "checkfilesforsequenceavail is already running -> exit" >> $scriptlog 2>&1
    56    exit
    57 fi
     62# check if script is already running
     63date > $lockpath/lock-checkfileavail.txt >> $scriptlog 2>&1
     64checklock0=$?
     65case $checklock0 in
     66    0)   echo "checklock0=$checklock0 -> continue " >> $scriptlog 2>&1;;
     67    1)   echo "checklock0=$checklock0 -> file exists " >> $scriptlog 2>&1
     68         echo "-> checkfileavail is running -> exit" >> $scriptlog 2>&1
     69         date  >> $scriptlog 2>&1
     70         exit;;
     71    *)   echo "checklock0=$checklock0 -> something went completely wrong" >> $scriptlog 2>&1;;
     72esac
    5873
     74# get todo list
    5975echo "getting list..." >> $scriptlog 2>&1
    6076check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    6278case $check0 in
    6379   1)   echo "check0=$check0 -> everything ok -> do step" >> $scriptlog 2>&1 ;;
    64    *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 ;;
     80   *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
     81        rm -v $todofile >> $scriptlog 2>&1
     82        rm -v $lockpath/lock-checkfileavail.txt >> $scriptlog 2>&1
     83        date  >> $scriptlog 2>&1
     84        exit;;
    6585esac
    6686
     87# retrieve sequences from todofile
    6788sequences=(`cat $todofile`)
    68 
    6989if [ "$sequences" = "" ]
    7090then
    7191   echo "nothing to do -> exit"  >> $scriptlog 2>&1
    7292   rm -v $todofile >> $scriptlog 2>&1
    73    rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     93   rm -v $lockpath/lock-checkfileavail.txt >> $scriptlog 2>&1
    7494   date  >> $scriptlog 2>&1
    7595   exit
    … …  
    7797
    7898echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    79 
    8099for sequence in ${sequences[@]}
    81100do
    … …  
    107126
    108127rm -v $todofile  >> $scriptlog 2>&1
    109 rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     128rm -v $lockpath/lock-checkfileavail.txt >> $scriptlog 2>&1
    110129
    111130date  >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/checkstardone

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the checking of the availability of the files for
     28# the datasets.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each dataset the sequences are extracted from the datasetfile
     33# and for each sequence the availability of the star files checked by using
     34# the macro checkstardone.C
     35# If this was successful, the status is inserted into the database using
     36# the macro setstatus.C
    2737#
    2838
    … …  
    4252scriptlogpath=$logpath/run/checkstardone/`date +%Y/%m/%d`
    4353makedir $scriptlogpath
    44 
    4554scriptlog=$scriptlogpath/checkstardone-$datetime.log
    4655
    … …  
    5160cd $mars
    5261
     62# check if script is already running
     63date > $lockpath/lock-checkstardone.txt >> $scriptlog 2>&1
     64checklock0=$?
     65case $checklock0 in
     66    0)   echo "checklock0=$checklock0 -> continue " >> $scriptlog 2>&1;;
     67    1)   echo "checklock0=$checklock0 -> file exists " >> $scriptlog 2>&1
     68         echo "-> checkstardone is running -> exit" >> $scriptlog 2>&1
     69         date  >> $scriptlog 2>&1
     70         exit;;
     71    *)   echo "checklock0=$checklock0 -> something went completely wrong" >> $scriptlog 2>&1;;
     72esac
    5373
    54 if [ -e $todofile ]
    55 then
    56    echo "checkstardone is already running -> exit" >> $scriptlog 2>&1
    57    exit
    58 fi
    59 
     74# get todo list
    6075echo "getting list..." >> $scriptlog 2>&1
    6176check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    6378case $check0 in
    6479   1)   echo "check0=$check0 -> everything ok -> do step" >> $scriptlog 2>&1 ;;
    65    *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1 ;;
     80   *)   echo "check0=$check0 -> ERROR -> could not get list -> exit" >> $scriptlog 2>&1
     81        rm -v $todofile >> $scriptlog 2>&1
     82        rm -v $lockpath/lock-checkstardone.txt >> $scriptlog 2>&1
     83        date  >> $scriptlog 2>&1
     84        exit;;
    6685esac
    6786
    6887datasets=(`cat $todofile`)
    69 
    7088if [ "$datasets" = "" ]
    7189then
    7290   echo "nothing to do -> exit"  >> $scriptlog 2>&1
    7391   rm -v $todofile >> $scriptlog 2>&1
    74    rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     92   rm -v $lockpath/lock-checkstardone.txt >> $scriptlog 2>&1
    7593   date  >> $scriptlog 2>&1
    7694   exit
    … …  
    7896
    7997echo "datasets: "${datasets[@]} #>> $scriptlog 2>&1
    80 
    8198for dataset in ${datasets[@]}
    8299do
    … …  
    123140
    124141rm -v $todofile  >> $scriptlog 2>&1
    125 rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     142rm -v $lockpath/lock-checkstardone.txt >> $scriptlog 2>&1
    126143
    127144date  >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/copyscript

    r7456 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    … …  
    2727#
    2828# This script handles the subsystem logfiles, which are copied every
    29 # day automatically from pc15 in La Palma to /data/MAGIC/transfer/ in
    30 # Wuerzburg
     29# day automatically from /local/ on pc15 in La Palma to
     30# /home/lapalma/transfer/ on coma in the datacenter in Wuerzburg
    3131# The structure in this directory is: subsystem/date/logfile.*
    3232# The logfiles are from CC (*.rep, CC*.dbg, CC*.rbk, CC*.rep, CC*.run,
    33 # CC*.run.html) and Cosy (cosy*.log, cosy*.rep, tpoint*.txt/starg*.txt)
    34 # AMC (AMC*.log, AMC*.info, AMC*.lut) and CaCo (dc*.txt) are not copied
    35 # every day automatically
     33# CC*.run.html), Cosy (cosy*.log, cosy*.rep, tpoint*.txt/starg*.txt)
     34# AMC (AMC*.log, AMC*.info, AMC*.lut) and CaCo (dc*.txt)
    3635#
    3736# the script performs the following steps:
    … …  
    4039# using the macros filldotrbk.C and filldotrun.C
    4140# - copying all logfiles to the correct directory:
    42 # /Period*/subsystem/date/file.*
    43 # (the period for each date is obtained by the macro getperiod.C)
    44 # - filling the sequences for the standard analysis into the database
    45 # (for each day of which a file was copied)
    46 #
     41# /subsystemdata/subsystem/YYYY/MM/DD/file.*
     42# - building the sequences for the standard analysis in the database
     43#   (for each day of which a file was copied)
     44# - if a new catalogfile has been copied, it is installed as standard in the
     45#   setup directory /magic/datacenter/setup and the new lines are sent to
     46#   the adresses specified in the script
    4747#
    4848# -- to make sure that no file is copied twice there's a textfile
    … …  
    6666transdir=/home/lapalma/transfer
    6767char=28
    68 #extern=MAGIC@virgo:/data/MAGIC/transfer
    6968datetime=`date +%F-%H-%M-%S`
    7069
    … …  
    7675date >> $scriptlog 2>&1
    7776
     77# check if the script is already running
    7878date > $lockpath/lock-copyscript.txt >> $scriptlog 2>&1
    7979checklock0=$?
    … …  
    8787esac
    8888
    89 # rsync new dc with virgo
    90 #rsync -av $extern/ $transdir  >> $scriptlog 2>&1
    91 
    9289cd $transdir
    9390
    9491#check the checksums, write output into file
    9592md5sum -c .checksums | tee $transdir/.check  >> $scriptlog 2>&1
    96 #md5sum -c $transdir/checksums-test >| $transdir/.check
     93
    9794if ! test -e $transdir/.check
    9895then
    … …  
    113110#dates=`ls $transdir/*/ -1 | grep --regexp=20[0-9][0-9]_*[0-1][0-9]_*[0-3][0-9] | grep -v '.tgz'`
    114111dates=`ls $transdir/*/ -1 | grep --regexp=20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] | grep -v '.tgz'` >> $scriptlog 2>&1
    115 
    116112if [ "$dates" = "" ]
    117113then
    … …  
    123119
    124120echo "dates found: "$dates  >> $scriptlog 2>&1
    125 
    126121#loop over dates
    127122for date in ${dates[@]}
    … …  
    138133   
    139134   date2=`echo $date | sed -e 's/_/\//g'`
    140 #   echo "date2=$date2"
    141135   copylogpath=$logpath/copyscript/$date2
    142136   echo "copylogpath: "$copylogpath >> $scriptlog 2>&1
    … …  
    148142      echo "file: "$file  >> $scriptlog 2>&1
    149143      filename=`basename $file`
    150 #      echo "filename: "$filename
    151144                       
    152145      #check for each file whether it has to be processed
    … …  
    179172               1) echo "checkfillrun=$checkfillrun -> everything ok " >> $scriptlog 2>&1
    180173                  echo "-> insert date in SequenceBuildStatus for $date" >> $scriptlog 2>&1
    181 #                  date3=`echo $date | sed -e 's/_/-/g'`
    182 #                  echo "date3=$date3"
     174
    183175                  checkinsertdate=`root -q -b $macrospath/insertdate.C+\("\"$date\""\) | tee $copylogpath/insertdate-$date-log.txt   | grep int | sed -e 's/(int)//'`
    184176                  case $checkinsertdate in
    … …  
    224216
    225217rm -v $transdir/.check >> $scriptlog 2>&1
    226 
    227 #rsync -av $transdir/ $extern >> $scriptlog 2>&1
    228 #rsync -av --delete $transdir/ $extern >> $scriptlog 2>&1
    229218
    230219

  • trunk/MagicSoft/Mars/datacenter/scripts/correcttime

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script runs the timing correction for all run, that need the timing
     28# correction
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each run the timing correction is done
     33# If this was successful, the status is inserted into the database using
     34# the macro setstatus.C
     35#
     36# This script was needed for the data between 06/2004 and 02/2005 (including
     37# this months).
    2738#
    2839
    … …  
    5667esac
    5768
     69# retrieving runs from todo file
    5870runs=(`cat $listpath/ToDo-$table-$column.txt`)
    59 
    6071if [ "$runs" = "" ]
    6172then
    … …  
    6677
    6778echo "runs: "${runs[@]}
    68 
    6979for run in ${runs[@]}
    7080do
    … …  
    8090  timecorrpath=$outpath"/timecorrlogs"
    8191  echo "timecorrpath: "$timecorrpath
    82   make dir $timecorrpath
     92  makedir $timecorrpath
    8393 
    8494  check1=0

  • trunk/MagicSoft/Mars/datacenter/scripts/dodatacheck

    r7329 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the datacheck for all runs, which are not yet
     28# checked
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each run the following steps are done:
     33#  - filldotraw.C
     34#  - sinope (once for data and once for calibration events)
     35#  - fillsinope
     36#  - reset ExclusionsDone, so that the result from the datacheck can be
     37#    taken into account when doexclusions is running the next time
     38# If this was successful, the status is inserted into the database using
     39# the macro setstatus.C
    2740#
    2841
    … …  
    4659scriptlogpath=$logpath/run/dodatacheck/`date +%Y/%m`
    4760makedir $scriptlogpath
    48 
    4961scriptlog=$scriptlogpath/dodatacheck-$datetime.log
    5062
    … …  
    5365makedir $getstatuslogpath >> $scriptlog 2>&1
    5466
     67# check if there are already todo files
    5568echo "checking if other todo-files are there" >> $scriptlog 2>&1
    5669if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
    5770then
    5871   echo "other file(s) on disk " >> $scriptlog 2>&1
    59    echo " -> choose one file and start calibrating" >> $scriptlog 2>&1
     72   echo " -> choose one file and start datacheck " >> $scriptlog 2>&1
    6073else
    6174   date > $lockpath/lock-getting-dodatacheck-list.txt >> $scriptlog 2>&1
    … …  
    7083   esac
    7184
    72    if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
    73    then
    74       echo "other file(s) on disk " >> $scriptlog 2>&1
    75       echo " -> choose one file and start datacheck" >> $scriptlog 2>&1
    76    else
    77       echo "getting list..." >> $scriptlog 2>&1
    78       check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    79 
    80       case $check0 in
    81           1)   echo "check0=$check0 -> everything ok -> do datacheck" >> $scriptlog 2>&1;;
    82           *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
    83       esac
    84    fi
     85   echo "getting list..." >> $scriptlog 2>&1
     86   check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
     87
     88   case $check0 in
     89      1)   echo "check0=$check0 -> everything ok -> do datacheck" >> $scriptlog 2>&1;;
     90      *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     91   esac
    8592   rm -v $lockpath/lock-getting-dodatacheck-list.txt >> $scriptlog 2>&1
    8693fi
    8794
    8895
     96# finding a todo file
    8997nr=bla
    9098echo "finding the right todo-file" >> $scriptlog 2>&1
    … …  
    92100
    93101echo "todofiles: "${todofiles[@]} >> $scriptlog 2>&1
    94 
    95102for todofile in ${todofiles[@]}
    96103do
    … …  
    120127
    121128
     129# get run(s) from todo file
    122130runs=(`cat $todofile`)
    123 
    124131if [ "$runs" = "" ]
    125132then
    … …  
    131138fi
    132139
     140# processing run(s)
    133141echo "runs: "${runs[@]} >> $scriptlog 2>&1
    134 
    135142for run in ${runs[@]}
    136143do
    … …  
    143150   date=`find /magic/data/rawfiles/ -name *${run}_[D,P,C,S]_*_E.raw | cut -c 22-31`
    144151   echo "date: "$date >> $scriptlog 2>&1
     152   # for sinope the date is needed in the format YYYY-MM-DD
    145153   date2=`echo $date | sed -e 's/\//-/g'`
    146154   echo "date2: "$date2 >> $scriptlog 2>&1
    147155
     156   # running filldotraw
    148157   filldotrawpath=$logpath/filldotraw/$date
    149158   makedir $filldotrawpath >> $scriptlog 2>&1
    … …  
    160169   esac
    161170   
     171   # running sinope
    162172   sinopepath=$datapath/sinope/$date
    163173   makedir $sinopepath >> $scriptlog 2>&1
    … …  
    183193   done
    184194   
     195   # running fillsinope
    185196   fillsinopepath=$logpath/fillsinope/$date
    186197   makedir $fillsinopepath >> $scriptlog 2>&1
    … …  
    191202
    192203   case $check3 in
    193        1)   echo "check3=$check3 -> everything ok -> set exclusions..." >> $scriptlog 2>&1
     204       1)   echo "check3=$check3 -> everything ok -> set status..." >> $scriptlog 2>&1
    194205            ;;
    195206       *)   echo "check3=$check3 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1
    … …  
    197208   esac
    198209
    199    echo "resetting the status for fExclusions done for date $date2" >> $scriptlog 2>&1
     210   # resetting the status for exclusions
     211   echo "resetting the status for fExclusionsDone for date $date2" >> $scriptlog 2>&1
    200212   resetlogpath=$logpath/resetexclusions/$date
    201213   echo "resetlogpath: $resetlogpath" >> $scriptlog 2>&1
    … …  
    212224   esac
    213225
     226   # set status
    214227   echo "inserting the status for the datacheck for run $run into the db" >> $scriptlog 2>&1
    215228   setstatuslogpath=$logpath/setstatus/datacheck/$date

  • trunk/MagicSoft/Mars/datacenter/scripts/doexclusions

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the excluding of runs for all days for which
     28# this step is needed.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then the run are excluded for each date in the todolist using the
     33# macro doexclusions.C
     34# If this was successful, the status is inserted into the database using
     35# the macro setstatus.C
    2736#
    2837
    … …  
    4655scriptlogpath=$logpath/run/doexclusions/`date +%Y/%m`
    4756makedir $scriptlogpath
    48 
    4957scriptlog=$scriptlogpath/doexclusions-$datetime.log
    5058
    … …  
    5361makedir $getstatuslogpath >> $scriptlog 2>&1
    5462
     63# check if the script is already running
    5564date > $lockpath/lock-doexclusions.txt >> $scriptlog 2>&1
    5665checklock0=$?
    … …  
    6473esac
    6574
    66 
     75# get the todo list
    6776echo "getting list..." >> $scriptlog 2>&1
    6877check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    7079case $check0 in
    7180   1)   echo "check0=$check0 -> everything ok -> do step" >> $scriptlog 2>&1;;
    72    *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     81   *)   echo "check0=$check0 -> ERROR -> could not get todo list -> exit" >> $scriptlog 2>&1
     82        rm -v $todofile >> $scriptlog 2>&1
     83        rm -v $lockpath/lock-doexclusions.txt >> $scriptlog 2>&1
     84        date  >> $scriptlog 2>&1
     85        exit;;
    7386esac
    7487
    7588dates=(`cat $todofile`)
    76 
    7789if [ "$dates" = "" ]
    7890then
    … …  
    8496fi
    8597
     98# do exclusions for each date in the todo file
    8699echo "dates: "${dates[@]} >> $scriptlog 2>&1
    87 
    88100for date in ${dates[@]}
    89101do

  • trunk/MagicSoft/Mars/datacenter/scripts/doqualityplots

    r7407 r7460  
    2020#   Author(s): Daniela Dorner  08/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script produces the qualityplots in the web by executing the macro
     28# plotdb.C
     29#
     30# After checking, if the script is already running, the plots are produced
     31# (ps and root file). Then they are moved to the webdirectory and there png
     32# files are produced from the ps file.
    2733#
    2834
    … …  
    4349date >> $scriptlog 2>&1
    4450
     51# check if script is already running
    4552lockfile=$lockpath/lock-doqualityplots.txt
    4653date > $lockfile >> $scriptlog 2>&1
    … …  
    5562esac
    5663
    57 
     64# producing the plots with the values from the database
    5865echo "producing plots: " >> $scriptlog 2>&1
    5966check0=`root -q -b $macrospath/plotdb.C+\(\) | tee -a $scriptlog | grep int | sed -e 's/.*(int)//'`
    … …  
    6673esac
    6774
    68 
     75# making files available in the web
    6976webpath=/www/htdocs/datacenter/datacheck
    7077name=plotdb
    … …  
    7481mv -v $name.{root,ps} $webpath  >> $scriptlog 2>&1
    7582
    76 echo "moving files: " >> $scriptlog 2>&1
     83echo "producing png files: " >> $scriptlog 2>&1
    7784pstoimg -antialias -flip r270 -density 100 -type png -multipage $psfile >> $scriptlog 2>&1
    7885

  • trunk/MagicSoft/Mars/datacenter/scripts/dowebplots

    r7454 r7460  
    2020#   Author(s): Daniela Dorner  08/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script produces the plots from all root-files in the web directory
     28#
     29# After checking, if the script is already running, the plots are produced:
     30# With the programm showplot a ps-file is written, from which the png
     31# files are produced.
    2732#
    2833
    … …  
    4449date >> $scriptlog 2>&1
    4550
     51# check if script is already running
    4652lockfile=$lockpath/lock-dowebplots.txt
    4753date > $lockfile >> $scriptlog 2>&1
    … …  
    6975   exit
    7076fi
    71 
    7277
    7378#produce plots for each rootfile
    … …  
    115120#make sure, that old plots of files, that are already removed from disk,
    116121#  are deleted also in the webdirectory
    117 #this has to be done, as ps and png files are excluded from rsync,
    118 #as rsync is done with --delete option (script /home/operator/condor/webupdate
     122#this has to be done, as ps and png files are excluded from rsync, and
     123#as rsync is done with --delete option (script /home/operator/condor/webupdate)
    119124
    120125#find all directories with plots

  • trunk/MagicSoft/Mars/datacenter/scripts/filesondisk

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
    27 #
     27# This script checks which files are on disk and updates the database
     28# accordingly. It is not yet running automatically.
     29#
     30# It is checking the:
     31# - ccfiles
     32# - cacofiles
     33#   this includes also the search for missing cacofiles:
     34#   Sometimes the DAQ aborts a run and starts itself a new one. In this
     35#   cases the camera controll doesn't start a new file, as the command to
     36#   start a new run was not sent by the central control. So the caco
     37#   information is stored in the previous caco file, which has a
     38#   different runnumber. To be able to merpp the information into the
     39#   calibrated data file, the runnumber of the file containing the
     40#   information has to be found.
     41#   In this script the search and inserting into the database is done
     42#   using the macros findcacofiles.C and insertcacofile.C
     43# - rawfiles
     44# The update in the database is done using the macro resetallruns.C
     45#
    2846
    2947user=`whoami`
    … …  
    3957
    4058echo "checking disk for ccfiles..."
    41 
    4259filename=$filesondisklogpath/ccfilesondisk-$date.txt
    4360column=fCCFileAvail
    44 
    4561find $subsystemdir/cc/ -name '*_S.rep' | cut -d_ -f2  > $filename
    46 
    4762
    4863echo "resetting runs..."
    … …  
    5570
    5671
    57 
    5872echo "checking disk for cacofiles..."
    59 
    6073filename=$filesondisklogpath/cacofilesondisk-$date.txt
    6174column=fCaCoFileAvail
    62 
    6375find $subsystemdir/caco/ -name '*.txt' | cut -d_ -f8 | grep [0-9] > $filename
    64 
    6576
    6677echo "resetting runs..."
    … …  
    7182   *)   echo "check0=$check0 -> ERROR -> something went wrong while resetting";;
    7283esac
    73 
    74 
    7584
    7685echo "checking missing cacofiles..."
    … …  
    8392
    8493missingcacoruns=(`cat $filesondisklogpath/findcacofiles-$date.txt`)
    85 
    8694for missingcacorun in ${missingcacoruns[@]}
    8795do
    … …  
    121129
    122130
    123 
    124131echo "checking disk for rawfiles..."
    125 
    126132filename=$filesondisklogpath/rawfilesondisk-$date.txt
    127133column=fRawFileAvail
    128 
    129134find $datapath -name '*.raw' | cut -d_ -f2 > $filename
    130135

  • trunk/MagicSoft/Mars/datacenter/scripts/fillcallisto

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the filling of the results from the calibration for
     28# all sequences of which the results are not yet in the database.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each sequence in the todo list the calibration results are
     33# filled into the table Calibration in the database using the macros
     34# fillcalib.C and fillsignal.C
     35# If this was successful, the status is inserted into the database using
     36# the macro setstatus.C
    2737#
    2838
    … …  
    4454scriptlogpath=$logpath/run/fillcallisto/`date +%Y/%m/%d`
    4555makedir $scriptlogpath
    46 
    4756scriptlog=$scriptlogpath/runfillcallisto-$datetime.log
    4857
    … …  
    5362cd $mars
    5463
     64# check if the script is already running
    5565date > $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
    5666checklock0=$?
    … …  
    6474esac
    6575
     76# get todo file
    6677echo "getting list..." >> $scriptlog 2>&1
    6778check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    6980case $check0 in
    7081    1)   echo "check0=$check0 -> everything ok -> run fillcallisto" >> $scriptlog 2>&1;;
    71     *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     82    *)   echo "check0=$check0 -> ERROR -> could not get list -> exit" >> $scriptlog 2>&1
     83         rm -v $todofile >> $scriptlog 2>&1
     84         rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     85         date  >> $scriptlog 2>&1
     86         exit;;
    7287esac
    7388
    7489sequences=(`cat $todofile`)
    75 
    7690if [ "$sequences" = "" ]
    7791then
    … …  
    8397fi
    8498
     99# fill information into the database for all sequences in the todo file
    85100echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    86 
    87101for sequence in ${sequences[@]}
    88102do

  • trunk/MagicSoft/Mars/datacenter/scripts/fillganymed

    r7340 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the filling of the results of ganymed into the db
     28# for all datasets of which the results are not yet in the database.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each dataset in the todo list the ganymed results are filled
     33# into the table Ganymed in the database using the macro fillganymed.C
     34# If this was successful, the status is inserted into the database using
     35# the macro setstatus.C
    2736#
    2837
    … …  
    5261cd $mars
    5362
     63# check if script is already running
    5464date > $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
    5565checklock0=$?
    … …  
    6373esac
    6474
     75# get todo list
    6576echo "getting list..." >> $scriptlog 2>&1
    6677check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    6879case $check0 in
    6980    1)   echo "check0=$check0 -> everything ok -> run fillganymed" >> $scriptlog 2>&1;;
    70     *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     81    *)   echo "check0=$check0 -> ERROR -> could not get list -> exit" >> $scriptlog 2>&1
     82         rm -v $todofile >> $scriptlog 2>&1
     83         rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     84         date  >> $scriptlog 2>&1
     85         exit;;
    7186esac
    7287
     88# retrieve datasets from todo file
    7389datasets=(`cat $todofile`)
    74 
    7590if [ "$datasets" = "" ]
    7691then
    … …  
    8297fi
    8398
     99# run fillganymed for datasets
    84100echo "datasets: "${datasets[@]} >> $scriptlog 2>&1
    85 
    86101for dataset in ${datasets[@]}
    87102do
    … …  
    110125           esac
    111126          ;;
    112       *)   echo "check2=$check2 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     127      *)  echo "check2=$check2 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
    113128   esac
    114129done

  • trunk/MagicSoft/Mars/datacenter/scripts/fillstar

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the filling of the results of star into the db
     28# for all sequences of which the results are not yet in the database.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then for each sequence in the todo list the star results are filled
     33# into the table Star in the database using the macro fillstar.C
     34# If this was successful, the status is inserted into the database using
     35# the macro setstatus.C
    2736#
    2837
    … …  
    5261cd $mars
    5362
     63# check if the script is already running
    5464date > $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
    5565checklock0=$?
    … …  
    6373esac
    6474
     75# get todo list
    6576echo "getting list..." >> $scriptlog 2>&1
    6677check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    6879case $check0 in
    6980    1)   echo "check0=$check0 -> everything ok -> run fillstar" >> $scriptlog 2>&1;;
    70     *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     81    *)   echo "check0=$check0 -> ERROR -> could not get list -> exit" >> $scriptlog 2>&1
     82         rm -v $todofile >> $scriptlog 2>&1
     83         rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     84         date  >> $scriptlog 2>&1
     85         exit;;
    7186esac
    7287
     88# retrieve sequences from todo file
    7389sequences=(`cat $todofile`)
    74 
    7590if [ "$sequences" = "" ]
    7691then
    … …  
    8297fi
    8398
     99# run fillstar for sequences
    84100echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    85 
    86101for sequence in ${sequences[@]}
    87102do

  • trunk/MagicSoft/Mars/datacenter/scripts/insertdatasets

    r7337 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the inserting of the datasets into the db.
     28#
     29# Extract information (dataset number, source name, comment, observation
     30# mode) from dataset file and insert it into the database in the table
     31# DataSets using the macro insertdataset.C
    2732#
    2833
    … …  
    5560esac
    5661
    57 
     62# get all datasetfiles
    5863datasetfiles=(`ls $datasetpath/*/*`)
    5964echo "datasetfiles: "${datasetfiles[@]}  >> $scriptlog 2>&1
    6065echo "" >> $scriptlog 2>&1
    6166
     67# extract information from dataset file and insert it into db with the macro insertdataset.C
    6268for datasetfile in ${datasetfiles[@]}
    6369do
     70   # get datasetnumber from filename
    6471   no=`echo $datasetfile | cut -d/ -f5 | cut -c8-99 | cut -d. -f1`
     72   # get datasetnumber from file and get it with 8 digits
    6573   no2=`grep 'AnalysisNumber:' $datasetfile | sed -e 's/AnalysisNumber://g' | sed -e 's/ //g'`
    6674   no3=`printf %08d $no2`
     75   # compare the two datasetnumber
     76   # continue only if number is the same
    6777   if [ "$no" = "$no3" ]
    6878   then
    … …  
    7383      continue
    7484   fi
     85   # get source name, comment and observation mode from dataset file
    7586   source=`grep 'SourceName:' $datasetfile | sed -e 's/SourceName://g' | sed -e 's/ //g' | sed -e 's/#//g'` >> $scriptlog 2>&1
    7687   comment=`grep 'Comment:' $datasetfile | sed -e 's/Comment://g'` >> $scriptlog 2>&1
    … …  
    94105   insertdatasetlog=$insertdatasetpath/insertdataset-$no3.log
    95106
     107   # insert information into db
    96108   check0=`root -q -b $macrospath/insertdataset.C+\("\"$no2\""\,"\"$source\""\,"\"$wobble\""\,"\"$comment\""\,kFALSE\) | tee $insertdatasetlog | grep int | sed -e 's/(int)//'`
    97109   case $check0 in

  • trunk/MagicSoft/Mars/datacenter/scripts/insertsequences

    r7370 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the inserting of sequences into the db.
     28# It is not used in the automatic processing of data, but only for
     29# inserting of manual written sequences.
    2730#
     31# Find manual written sequence files and insert information into the
     32# database using the macro insertsequence.C
     33#
    2834
    2935user=`whoami`
    … …  
    4450date >> $scriptlog 2>&1
    4551
     52# check if script is already running
    4653date > $lockpath/lock-insertsequences.txt >> $scriptlog 2>&1
    4754checklock0=$?
    … …  
    5562esac
    5663
    57 
     64# finding manual written sequence files (marked by the comment '#manually changed')
    5865sequencefiles=(`grep -R '#manually changed' $sequpath/ | cut -c 1-42`)
    5966echo "sequencefiles: "${sequencefiles[@]}  >> $scriptlog 2>&1
    6067echo "" >> $scriptlog 2>&1
    6168
     69# process sequencefiles: insert information into the database
    6270for sequencefile in ${sequencefiles[@]}
    6371do

  • trunk/MagicSoft/Mars/datacenter/scripts/linkmc

    r7453 r7460  
    2020#   Author(s): Daniela Dorner  12/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script is linking the montecarlo files from the original structure
     28# to a structure from which the files can be processed more easily with the
     29# automatic analysis.
     30# This script is not yet running automatically.
     31#
     32# original structure:
     33# /montecarlo/camera/
     34#
     35# new structure:
     36# /montecarlo/rawfiles/YYYY/MM/DD/file.root
     37# more explanation concerning the file structure can be found in the file
     38# /montecarlo/rawfiles/README.txt
     39#
     40# First the data files are linked and then in each new directory also the
     41# pedestal and calibration file is linked.
    2742#
    2843
    … …  
    3853date >> $scriptlog 2>&1
    3954
     55# check if script is already running
    4056lockfile=$lockpath/lock-mclinks.txt
    4157date > $lockfile >> $scriptlog 2>&1
    … …  
    5066esac
    5167
    52 
    5368mccampath=/montecarlo/camera
    5469mcpath=/montecarlo/rawfiles
    … …  
    5772readme=$mcpath/README.txt #file in which the information about the modes is redirected to have always an updated explanation
    5873
     74# check if file with next runnumber is available
    5975if ! ls $next >> $scriptlog 2>&1
    6076then
    … …  
    6480fi
    6581
    66 date >| $readme 2>&1
    67 
    68 
     82
     83# observation modes
    6984modes=("" "Gammawobble+" "Gammanowobble0" "GammawobbleHE+" "GammanowobbleHE0" "Gammawobble0" "GammawobbleHE0" "Gammadiffuse0" "Protonnowobble0" )
    70 
    7185#be carful:
    7286# w- not yet foreseen in this script
    7387
     88# print information and explanation of structure into README.txt
     89date >| $readme 2>&1
    7490echo "" >> $readme 2>&1
    7591echo "Explanation for the structure in which the mc files are linked" >> $readme 2>&1
    … …  
    218234done
    219235
    220 
    221236rm -v $lockfile >> $scriptlog 2>&1
    222237
    … …  
    225240date  >> $scriptlog 2>&1
    226241
    227 

  • trunk/MagicSoft/Mars/datacenter/scripts/makecallistolinks

    r7281 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script links a special callisto.rc for data, for which the
     28# calibration with the standard callisto.rc failes.
     29#
     30# The sequences with failing calibration are searched by searching for the
     31# reason of failure in the callisto.log Then the special callisto.rc is
     32# linked to the directory of the sequence.
    2733#
    2834
    … …  
    4147date >> $scriptlog 2>&1
    4248
     49# check if script is already running
    4350lockfile=$lockpath/lock-making-callisto-links.txt
    4451date > $lockfile >> $scriptlog 2>&1
    … …  
    5360esac
    5461
    55 
     62# reason why calibration with standard callisto.rc fails
    5663reason="Pulse is too much to the right, cannot go beyond logain limits!"
    5764echo "reason: "$reason >> $scriptlog 2>&1
    5865
     66# find files containing reason
    5967files=`find $datapath/callisto -name callisto*.log`
    6068
     69# make links for callisto.rc for which the callisto.log contains reason
    6170for file in $files
    6271do

  • trunk/MagicSoft/Mars/datacenter/scripts/mcsequences

    r7448 r7460  
    2020#   Author(s): Daniela Dorner  12/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
    27 #
     27# This script is an intermediate solution to process the montecarlo files.
     28# As soon as the montecarlo database is running, this script shall be
     29# exchanged.
     30#
     31# The script is building one sequence per directory. As the files are
     32# structured by condition, the files in one directory and such in one
     33# sequence have the same observation mode, psf, zbin, ...
     34#
    2835
    2936user=`whoami`
    … …  
    3845date >> $scriptlog 2>&1
    3946
     47# check if script is already running
    4048lockfile=$lockpath/lock-mcsequences.txt
    4149date > $lockfile >> $scriptlog 2>&1
    … …  
    5462mcsequpath=/montecarlo/sequences
    5563
     64# find montecarlo directories, build one sequence per directory and write sequence file
    5665dirs=`find $mcpath -type d`
    57 
    5866for dir in ${dirs[@]}
    5967do

  • trunk/MagicSoft/Mars/datacenter/scripts/movingrawfiles

    r7426 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
     26#
     27# This script is moving the rawfiles from the directory
     28# /magic/datacenter/fromtape/rawdata to the structure
     29# /magic/data/rawfiles/YYYY/MM/DD
     30#
    2631
    2732umask 0002
    2833
     34# change permission for files
     35# perhaps needed as the tapes are read by the user tape
    2936ssh tape@dc07 chmod -R g+w /magic/datacenter/fromtape/rawdata/*
    3037# output for chmod with -v or -c (only changes)
     38# for files transferes via internet perhaps also needed for user lapalma
    3139
     40# find rawfiles
    3241rawfiles=`find /magic/datacenter/fromtape/rawdata/ -name '*.*'`
    33 
    3442for rawfile in $rawfiles
    3543do
    36 #  echo "file: "$rawfile
     44# workaround for rawfiles with wrong timing
    3745#  newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles-wrong-timing/g' -e 's/_/\//1' -e 's/_/\//1'`
    3846  newrawfile=`echo $rawfile | sed -e 's/center\/fromtape\/rawdata/\/rawfiles/g' -e 's/_/\//1' -e 's/_/\//1'`
    39 #  echo "new file: "$newrawfile
    4047  newdir=`dirname $newrawfile`
    41 #  echo "newdir: "$newdir
    4248  if [ ! -d $newdir ]
    4349  then
    … …  
    4753
    4854  mv -v $rawfile $newrawfile
    49 
    5055done
    5156

  • trunk/MagicSoft/Mars/datacenter/scripts/movingsubsystemfiles

    r7426 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
     26#
     27# This script is moving the subsystemfiles from the directory
     28# /magic/datacenter/fromtape to the structure
     29# /magic/subsystemdata/subsystem/YYYY/MM/DD
     30#
    2631
    2732umask 0002
    2833
     34# change permission for files
     35# perhaps needed as the tapes are read by the user tape
    2936ssh tape@dc07 chmod -R g+w /magic/datacenter/fromtape/{ccdata,cacodata,amc,daqlog}/*
    3037
     38# find subsystem files
    3139subsysfiles=`find /magic/datacenter/fromtape/ccdata/ -name '*.*'`
    3240subsysfiles=("${subsysfiles[@]}" `find /magic/datacenter/fromtape/daqlog/ -name '*.*'`)
    … …  
    3442subsysfiles=("${subsysfiles[@]}" `find /magic/datacenter/fromtape/amc/ -name '*.*'`)
    3543
     44# moving subsystem files
    3645for subsysfile in ${subsysfiles[@]}
    3746do
    38 #  echo "file: "$subsysfile
    3947  newsubsysfile=`echo $subsysfile | sed -e 's/datacenter\/fromtape/subsystemdata/g' -e 's/daqlog/daq__/g' -e 's/ccdata/cc/g' -e 's/cacodata/caco/g' -e 's/_/\//1' -e 's/_/\//1' -e 's/\/\///g'`
    40 #  echo "new file: "$newsubsysfile
    4148  newdir=`dirname $newsubsysfile`
    42 #  echo "newdir: "$newdir
    4349  if [ ! -d $newdir ]
    4450  then
    … …  
    4854
    4955  mv -v $subsysfile $newsubsysfile
    50 
    5156done
    5257

  • trunk/MagicSoft/Mars/datacenter/scripts/processmcsequences

    r7448 r7460  
    2020#   Author(s): Daniela Dorner  12/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script is an intermediate solution to process the montecarlo files.
     28# As soon as the montecarlo database is running, this script shall be
     29# exchanged.
     30#
     31# The script is processing the montecarlo sequences: callisto and star
    2732#
    2833
    … …  
    3843date >> $scriptlog 2>&1
    3944
     45# check if script is already running
    4046lockfile=$lockpath/lock-processmcsequences.txt
    4147date > $lockfile >> $scriptlog 2>&1
    … …  
    5056esac
    5157
    52 
    5358mcpath=/montecarlo/rawfiles
    5459mccalpath=/montecarlo/callisto
    … …  
    5762callistorc=$setuppath/callisto/callisto.rc
    5863
    59 sequfiles=`find $mcsequpath -type f`
    60 
    6164cd $mars
    6265
     66# find montecarlo sequence files
     67sequfiles=`find $mcsequpath -type f`
    6368for sequfile in ${sequfiles[@]}
    6469do
    … …  
    6974   makedir $caloutpath >> $scriptlog 2>&1
    7075   
     76   # continue with next sequence, if sequence is already processed
     77   #  or processing  of sequence is just running
    7178   if ls $caloutpath/.done >> $scriptlog 2>&1
    7279   then
    7380      continue
    7481   fi
    75    
    7682   if ls $caloutpath/.doing >> $scriptlog 2>&1
    7783   then
    … …  
    7985   fi
    8086   
     87   # lock file
    8188   touch $caloutpath/.doing
    82    
    8389   echo "processing sequ $sequfile" >> $scriptlog 2>&1
    84    
     90   # find callisto.rc file
    8591   if [ -e $caloutpath/callisto.rc ]
    8692   then
    … …  
    9298      callistorcseq=$caloutpath/callisto.rc
    9399   fi
    94    
     100   # calibrate data
    95101   echo "./callisto -mc -f -b -q --ind=$mcpath/$date --log=$caloutpath/callisto$sequno.log --html=$caloutpath/callisto$sequno.html --out=$caloutpath --config=$callistorcseq $sequfile 2>> $scriptlog > /dev/null" >> $scriptlog 2>&1
    96102   ./callisto -mc -f -b -q --ind=$mcpath/$date --log=$caloutpath/callisto$sequno.log --html=$caloutpath/callisto$sequno.html --out=$caloutpath --config=$callistorcseq $sequfile 2>> $scriptlog > /dev/null
    97103   check1=$?
    98  
     104   # if calibration worked -> run star for sequence
    99105   case $check1 in
    100106      0)   echo "check1=$check1 -> everything ok..." >> $scriptlog 2>&1
    … …  
    103109           imgoutpath=$mccalpath/$no/$sequno
    104110           makedir $imgoutpath >> $scriptlog 2>&1
     111           
    105112           ./star -b -q -v4 -f -mc --log=$imgoutpath/star$sequno.log --html=$imgoutpath/star$sequno.html --ind=$caloutpath --out=$imgoutpath $sequfile  2>> $scriptlog> /dev/null
    106113           check2=$?
    … …  
    118125done
    119126
    120 
    121127rm -v $lockfile >> $scriptlog 2>&1
    122128

  • trunk/MagicSoft/Mars/datacenter/scripts/rmlocks

    r7452 r7460  
    2020#   Author(s): Daniela Dorner  08/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script is removing lock files which are older than 12 hours.
     28#
     29# Each process in the automatic analysis is writing a lock file. As these
     30# processes are launched with condor, they are stopped after 12 hours, if
     31# they are still running. In this case the lock files are not removed.
     32#
     33# By comparing the date of the lock file and the current date, the
     34# running time of a process is determined and if it is bigger than 12
     35# hours (which means that the process has been already kill by condor) the
     36# lock file is removed
    2737#
    2838

  • trunk/MagicSoft/Mars/datacenter/scripts/runcallisto

    r7426 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script is launching the calibration of sequences.
     28#
     29# In the case of calibration only one sequence is processed. Despite of
     30# that the structure of the script is such, that also more sequences could
     31# be processed. The restriction to one sequence has been made, as the
     32# calibration takes some time. There's one todo file per sequence.
     33# First the script searches for a todo file. Then the sequence from this
     34# todo file is calibrated and the merpp update is done.
     35#
     36# the callisto.rc files are stored in the setup directory
    2737#
    2838
    … …  
    3242set -C
    3343
     44# define callisto.rc files
    3445callistorcnew=$setuppath/callisto/callisto.rc
    3546callistorcmarapr05=$setuppath/callisto/callisto_MarApr05.rc
    … …  
    5667cd $mars
    5768
     69# get todo file
    5870echo "checking if other todo-files are there" >> $scriptlog 2>&1
    5971if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
    … …  
    7385   esac
    7486
    75    if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
    76    then
    77       echo "other file(s) on disk " >> $scriptlog 2>&1
    78       echo " -> choose one file and start calibrating" >> $scriptlog 2>&1
    79    else
    80       echo "getting list..." >> $scriptlog 2>&1
    81       check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    82 
    83       case $check0 in
    84           1)   echo "check0=$check0 -> everything ok -> run callisto" >> $scriptlog 2>&1;;
    85           *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
    86       esac
    87    fi
     87   echo "getting list..." >> $scriptlog 2>&1
     88   check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
     89
     90   case $check0 in
     91      1)   echo "check0=$check0 -> everything ok -> run callisto" >> $scriptlog 2>&1;;
     92      *)   echo "check0=$check0 -> ERROR -> could not get todo list -> exit" >> $scriptlog 2>&1
     93           rm -v $lockpath/lock-getting-callisto-list.txt >> $scriptlog 2>&1
     94           date  >> $scriptlog 2>&1
     95           exit;;
     96   esac
    8897   rm -v $lockpath/lock-getting-callisto-list.txt >> $scriptlog 2>&1
    8998fi
    9099
    91 
     100# choose todo file
    92101nr=bla
    93102echo "finding the right todo-file" >> $scriptlog 2>&1
    94103todofiles=`ls -r $listpath/ToDo-$table-$column-*`
    95 
    96 #echo "todofiles: "${todofiles[@]} >> $scriptlog 2>&1
    97104
    98105for todofile in ${todofiles[@]}
    … …  
    122129esac
    123130
     131# get sequence(s) from todo file
    124132sequences=(`cat $todofile`)
    125 
    126133if [ "$sequences" = "" ]
    127134then
    … …  
    133140fi
    134141
     142# run calibration for sequence(s)
    135143echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    136 
    137144for sequence in ${sequences[@]}
    138145do
    … …  
    146153  echo "sequfile: "$sequfile >> $scriptlog 2>&1
    147154 
     155  # find callisto.rc file
    148156  if [ -e $outpath/callisto.rc ]
    149157  then
    … …  
    167175  case $check1 in
    168176     0)   echo "check1=$check1 -> everything ok -> doing update..." >> $scriptlog 2>&1
     177          # running merpp update if calibration worked
     178          # finding files, which have to be updated
    169179          echo "finding files to be updated" >> $scriptlog 2>&1
    170180          calfiles=`find $outpath -name *_Y_* `
    … …  
    179189          makedir $merpplogpath >> $scriptlog 2>&1
    180190         
     191          # updated calibrated data files with the information from the cc and caco files
    181192          for calfile in ${calfiles[@]}
    182193          do
    183194             echo "calfile: "$calfile >> $scriptlog 2>&1
     195             # find cc and caco file
     196             # if file is missing continue with next sequence
    184197             runno=`echo $calfile | cut -d_ -f2 | sed -e 's/^0//' | sed -e 's/^0//' | sed -e 's/^0//' `
    185198             ccfile=`find /magic/subsystemdata/cc/ -name [2][0][0-2][0-9][0,1][0-9][0-3][0-9]_*${runno}_[P,D,C,S]_*_S.rep`
    … …  
    232245          done
    233246         
     247          # set status for calibration if merpp updates are also done
    234248          echo "inserting the status for callisto for sequence $sequence into the db" >> $scriptlog 2>&1
    235249          setstatuslogpath=$logpath/setstatus/callisto/$no

  • trunk/MagicSoft/Mars/datacenter/scripts/runganymed

    r7426 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script is launching ganymed for datasets.
     28#
     29# In the case of ganymed only one dataset is processed. Despite of
     30# that the structure of the script is such, that also more datasets could
     31# be processed. The restriction to one dataset has been made, as ganymed
     32# takes some time. There's one todo file per dataset.
     33# First the script searches for a todo file. Then ganymed is run for the
     34# dataset from this todo file.
     35#
     36# the ganymed.rc files are stored in the setup directory
    2737#
    2838
    … …  
    4555scriptlogpath=$logpath/run/ganymed/`date +%Y/%m/%d`
    4656makedir $scriptlogpath
    47 
    4857scriptlog=$scriptlogpath/runganymed-$datetime.log
    4958
    … …  
    5463cd $mars
    5564
     65# check if getting of list is already running
    5666date > $lockpath/lock-getting-ganymed-list.txt >> $scriptlog 2>&1
    5767checklock0=$?
    … …  
    6575esac
    6676
     77# finding todo file
    6778echo "checking if other todo-files are there" >> $scriptlog 2>&1
    6879if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
    … …  
    8293rm -v $lockpath/lock-getting-ganymed-list.txt >> $scriptlog 2>&1
    8394
     95# choosing todo file
    8496nr=bla
    8597echo "finding the right todo-file" >> $scriptlog 2>&1
    8698todofiles=`ls $listpath/ToDo-$table-$column-*`
    87 
    88 #echo "todofiles: "${todofiles[@]} >> $scriptlog 2>&1
    8999
    90100for todofile in ${todofiles[@]}
    … …  
    114124esac
    115125
    116 
     126# retrieve dataset from todo file
    117127datasets=(`cat $todofile`)
    118 
    119128if [ "$datasets" = "" ]
    120129then
    … …  
    126135fi
    127136
     137# run ganymed for dataset
    128138echo "datasets: "${datasets[@]} >> $scriptlog 2>&1
    129 
    130139for dataset in ${datasets[@]}
    131140do
    … …  
    138147  datasetfile="$datasetpath/$no/dataset$no2.txt"
    139148  echo "datasetfile: "$datasetfile >> $scriptlog 2>&1
     149  # get observation mode to choose ganymed.rc file
    140150  wobble=`grep 'WobbleMode:' $datasetfile` >> $scriptlog 2>&1
    141151  wobble2=`echo $wobble | grep ^\#` >> $scriptlog 2>&1

  • trunk/MagicSoft/Mars/datacenter/scripts/runstar

    r7426 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script is launching star for sequence.
     28#
     29# In the case of star only one sequence is processed. Despite of
     30# that the structure of the script is such, that also more sequences could
     31# be processed. The restriction to one sequence has been made, as star
     32# takes some time. There's one todo file per sequence.
     33# First the script searches for a todo file. Then star is run for the
     34# sequence from this todo file.
     35#
     36# the star.rc files are taken from the mars directory
    2737#
    2838
    … …  
    5363cd $mars
    5464
     65# check if getting of list is already running
    5566date > $lockpath/lock-getting-star-list.txt >> $scriptlog 2>&1
    5667checklock0=$?
    … …  
    6475esac
    6576
     77# finding todo file
    6678echo "checking if other todo-files are there" >> $scriptlog 2>&1
    6779if  ls $todofile-[1-9]*.txt >> $scriptlog 2>&1
    … …  
    8193rm -v $lockpath/lock-getting-star-list.txt >> $scriptlog 2>&1
    8294
     95# choosing todo file
    8396nr=bla
    8497echo "finding the right todo-file" >> $scriptlog 2>&1
    8598todofiles=`ls $listpath/ToDo-$table-$column-*`
    86 
    87 #echo "todofiles: "${todofiles[@]} >> $scriptlog 2>&1
    8899
    89100for todofile in ${todofiles[@]}
    … …  
    114125
    115126
     127# retrieve sequence from todo file
    116128sequences=(`cat $todofile`)
    117 
    118129if [ "$sequences" = "" ]
    119130then
    … …  
    125136fi
    126137
     138# run star for sequence
    127139echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    128 
    129140for sequence in ${sequences[@]}
    130141do

  • trunk/MagicSoft/Mars/datacenter/scripts/sourcefile

    r7233 r7460  
    2020#   Author(s): Daniela Dorner  05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This a resource file for the scripts, in which the standard paths and
     28# functions, which are needed more often are stored.
    2729#
    2830

  • trunk/MagicSoft/Mars/datacenter/scripts/writesequencefiles

    r7265 r7460  
    2020#   Author(s): Daniela Dorner  08/2004 <mailto:dorner@astro.uni-wuerzburg.de>
    2121#
    22 #   Copyright: MAGIC Software Development, 2000-2004
     22#   Copyright: MAGIC Software Development, 2000-2006
    2323#
    2424#
    2525# ========================================================================
    2626#
     27# This script launches the writing of the sequence files for all sequences
     28# for which this step is needed.
     29#
     30# After checking, if the script is already running, the todolist is
     31# written by using the macro getdolist.C
     32# Then the sequence file is written for each sequence in the todolist using
     33# the macro writesequencefile.C
     34# If this was successful, the status is inserted into the database using
     35# the macro setstatus.C
    2736#
    2837
    … …  
    5261cd $mars
    5362
     63# check if script is already running
    5464date > $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
    5565checklock0=$?
    … …  
    6373esac
    6474
     75# get todo list
    6576echo "getting list..." >> $scriptlog 2>&1
    6677check0=`root -q -b $macrospath/getdolist.C+\("\"$table\""\,"\"$column\""\,"\"$date\""\,"\"$listpath\""\) | tee $getstatuslog | grep int | sed -e 's/(int)//'`
    … …  
    6879case $check0 in
    6980    1)   echo "check0=$check0 -> everything ok -> run writesequencefiles" >> $scriptlog 2>&1;;
    70     *)   echo "check0=$check0 -> ERROR -> step has to be repeated" >> $scriptlog 2>&1;;
     81    *)   echo "check0=$check0 -> ERROR -> could not get todo list -> exit" >> $scriptlog 2>&1
     82         rm -v $todofile >> $scriptlog 2>&1
     83         rm -v $lockpath/lock-$table-$column.txt >> $scriptlog 2>&1
     84         date  >> $scriptlog 2>&1
     85         exit;;
    7186esac
    7287
     88# get sequences from todo file
    7389sequences=(`cat $todofile`)
    74 
    7590if [ "$sequences" = "" ]
    7691then
    … …  
    8297fi
    8398
     99# write sequence file for all sequences in todo file
    84100echo "sequences: "${sequences[@]} >> $scriptlog 2>&1
    85 
    86101for sequence in ${sequences[@]}
    87102do
Note: See TracChangeset for help on using the changeset viewer.