Index: trunk/MagicSoft/Mars/datacenter/scripts/mcsequences =================================================================== --- trunk/MagicSoft/Mars/datacenter/scripts/mcsequences (revision 8763) +++ trunk/MagicSoft/Mars/datacenter/scripts/mcsequences (revision 8967) @@ -19,17 +19,14 @@ # # Author(s): Daniela Dorner 12/2005 -# -# Copyright: MAGIC Software Development, 2000-2007 +# Author(s): Daniela Dorner 06/2008 +# +# Copyright: MAGIC Software Development, 2000-2008 # # # ======================================================================== # -# This script is an intermediate solution to process the montecarlo files. -# As soon as the montecarlo database is running, this script shall be -# exchanged. -# -# The script is building one sequence per directory. As the files are -# structured by condition, the files in one directory and such in one -# sequence have the same observation mode, psf, zbin, ... +# The script is building two sequences per directory and epoch. One for +# training and one for testing. The relevant parameters are taken from +# the MC database. # @@ -46,17 +43,20 @@ lockfile=$lockpath/lock-$program.txt checklock >> $scriptlog 2>&1 - -# find montecarlo directories, build one sequence per directory and write sequence file -printprocesslog "INFO building one sequence per mcdirectory and epoch" - -user="test" -db="TestMonteCarlo" -pw="Ics+eaTD" + +# find montecarlo directories, build two sequences per directory and write sequence files +printprocesslog "INFO building two sequences per mcdirectory and epoch, one for training and one for testing" + +#user="test" +#db="TestMonteCarlo" +#pw="Ics+eaTD" +user="operator" +db="MyMagicMonteCarlo" +pw="Ics+eMM!" dirs=`find $mcrawpath -type d` for dir in ${dirs[@]} do - cont=`echo $dir | cut -d/ -f6` -# cont=`echo $dir | cut -d/ -f10` + cont=`echo $dir | cut -d/ -f7` + #cont=`echo $dir | cut -d/ -f10` if [ "$cont" == "" ] then @@ -65,5 +65,5 @@ echo $dir >> $scriptlog 2>&1 - epochs=`ls -l $mcpath/camera | cut -c 52-70` + epochs=`ls -l $mcpath/camera | cut -c 52-80` for epoch in ${epochs[@]} do @@ -81,63 +81,146 @@ if [ "$datruns" != "" ] then + num=`echo $datruns | wc -w` + train=`expr $num / 5` + test=`expr $num - $train` + last=`expr $num \* 9` + train2=`expr $train \* 9` firstrun=`echo $datruns | cut -c 0-8` + trainruns=`echo $datruns | cut -c 0-$train2` + trainruns2=`echo $trainruns | sed -e 's/ /,/g'` + testruns=`echo $datruns | cut -c $train2-$last` + testruns2=`echo $testruns | sed -e 's/ /,/g'` + firstruntrain=`echo $trainruns | cut -c 0-8` + firstruntest=`echo $testruns | cut -c 0-8` + date=`echo $dir | cut -c 28-37 | sed -e 's/\//-/g'` #date=`echo $dir | cut -c 57-66 | sed -e 's/\//-/g'` zbin=`echo $date | cut -c 3,4` - - for datrun in ${datruns[@]} + + + #training runs + for trainrun in ${trainruns[@]} do - update=" update MCRunData set fSequenceFirst=\"$firstrun\" where fRunNumber=\"$datrun\" " - mysql -ss -u $user --password=$pw --host=vela $db -e " $update " + trainupdate=" update MCRunData set fSequenceFirst=\"$firstruntrain\" where fRunNumber=\"$trainrun\" " + mysql -ss -u $user --password=$pw --host=vela $db -e " $trainupdate " done - query=" select fSpectrumKEY, fPSFKEY, fAmplFadcKEY, fAtmosphericModelKEY, fObservationModeKEY, fMCParticleKEY, fReflectorVersionKEY, fCorsikaVersionKEY, fCameraVersionKEY, fViewConeAngleOKEY from MCRunData where fRunNumber=\"$firstrun\" " - values=`mysql -ss -u $user --password=$pw --host=vela $db -e " $query "` - spec=`echo $values | cut -d" " -f1` - psf=`echo $values | cut -d" " -f2` - amplfadc=`echo $values | cut -d" " -f3` - atmo=`echo $values | cut -d" " -f4` - obs=`echo $values | cut -d" " -f5` - part=`echo $values | cut -d" " -f6` - refl=`echo $values | cut -d" " -f7` - cors=`echo $values | cut -d" " -f8` - cam=`echo $values | cut -d" " -f9` - view=`echo $values | cut -d" " -f10` - query1=" select fSequenceFirst from MCSequenceProcessStatus where fSequenceFirst=\"$firstrun\" " - query2=" select fSequenceFirst from MCSequences where fSequenceFirst=\"$firstrun\" " - check1=`mysql -ss -u $user --password=$pw --host=vela $db -e " $query1 "` - check2=`mysql -ss -u $user --password=$pw --host=vela $db -e " $query2 "` - insert1=" insert MCSequenceProcessStatus set fSequenceFirst=\"$firstrun\", fSequenceFileWritten=Now() " - insert2=" insert MCSequences set fSequenceFirst=\"$firstrun\", fRunStart=\"$date\", fZBin=\"$zbin\", fSpectrumKEY=\"$spec\", fPSFKEY=\"$psf\", fAmplFadcKEY=\"$amplfadc\", fAtmosphericModelKEY=\"$atmo\", fObservationModeKEY=\"$obs\", fMCParticleKEY=\"$part\", fReflectorVersionKEY=\"$refl\", fCorsikaVersionKEY=\"$cors\", fCameraVersionKEY=\"$cam\", fViewConeAngleOKEY=\"$view\" " - update2=" update MCSequences set fRunStart=\"$date\", fZBin=\"$zbin\", fSpectrumKEY=\"$spec\", fPSFKEY=\"$psf\", fAmplFadcKEY=\"$amplfadc\", fAtmosphericModelKEY=\"$atmo\", fObservationModeKEY=\"$obs\", fMCParticleKEY=\"$part\", fReflectorVersionKEY=\"$refl\", fCorsikaVersionKEY=\"$cors\", fCameraVersionKEY=\"$cam\", fViewConeAngleOKEY=\"$view\" where fSequenceFirst=\"$firstrun\" " - if [ "$check1" == "" ] - then - mysql -ss -u $user --password=$pw --host=vela $db -e " $insert1 " - fi - if [ "$check2" == "" ] - then - mysql -ss -u $user --password=$pw --host=vela $db -e " $insert2 " + trainquery1=" select fSpectrumKEY, fPSFKEY, fAmplFadcKEY, fAtmosphericModelKEY, fObservationModeKEY, fMCParticleKEY, fReflectorVersionKEY, fCorsikaVersionKEY, fCameraVersionKEY, fViewConeAngleOKEY, fAddSpotSizeKEY from MCRunData where fRunNumber=\"$firstruntrain\" " + trainquery2=" select sum(fNumEvents), sum(fNumtriggers) from MCRunData where fRunNumber in ($trainruns2) " + trainvalues=`mysql -ss -u $user --password=$pw --host=vela $db -e " $trainquery1 "` + trainvalues2=`mysql -ss -u $user --password=$pw --host=vela $db -e " $trainquery2 "` + trainspec=`echo $trainvalues | cut -d" " -f1` + trainpsf=`echo $trainvalues | cut -d" " -f2` + trainamplfadc=`echo $trainvalues | cut -d" " -f3` + traintrainatmo=`echo $trainvalues | cut -d" " -f4` + trainobs=`echo $trainvalues | cut -d" " -f5` + trainpart=`echo $trainvalues | cut -d" " -f6` + trainrefl=`echo $trainvalues | cut -d" " -f7` + traincors=`echo $trainvalues | cut -d" " -f8` + traincam=`echo $trainvalues | cut -d" " -f9` + trainview=`echo $trainvalues | cut -d" " -f10` + trainass=`echo $trainvalues | cut -d" " -f11` + trainevents=`echo $trainvalues2 | cut -d" " -f1` + traintriggers=`echo $trainvalues2 | cut -d" " -f2` + trainquery3=" select fSequenceFirst from MCSequenceProcessStatus where fSequenceFirst=\"$firstruntrain\" " + trainquery4=" select fSequenceFirst from MCSequences where fSequenceFirst=\"$firstruntrain\" " + traincheck3=`mysql -ss -u $user --password=$pw --host=vela $db -e " $trainquery3 "` + traincheck4=`mysql -ss -u $user --password=$pw --host=vela $db -e " $trainquery4 "` + traininsert1=" insert MCSequenceProcessStatus set fSequenceFirst=\"$firstruntrain\", fSequenceFileWritten=Now() " + traininsert2=" insert MCSequences set fSequenceFirst=\"$firstruntrain\", fRunStart=\"$date\", fNumEvents=\"$trainevents\", fNumTriggers=\"$traintriggers\", fZBin=\"$zbin\", fSpectrumKEY=\"$trainspec\", fPSFKEY=\"$trainpsf\", fAddSpotSizeKEY=\"$trainass\", fAmplFadcKEY=\"$trainamplfadc\", fAtmosphericModelKEY=\"$trainatmo\", fObservationModeKEY=\"$trainobs\", fMCParticleKEY=\"$trainpart\", fReflectorVersionKEY=\"$trainrefl\", fCorsikaVersionKEY=\"$traincors\", fCameraVersionKEY=\"$traincam\", fViewConeAngleOKEY=\"$trainview\", fTestTrainKEY=2 " + trainupdate2=" update MCSequences set fRunStart=\"$date\", fNumEvents=\"$trainevents\", fNumTriggers=\"$traintriggers\", fZBin=\"$zbin\", fSpectrumKEY=\"$trainspec\", fPSFKEY=\"$trainpsf\", fAddSpotSizeKEY=\"$trainass\", fAmplFadcKEY=\"$trainamplfadc\", fAtmosphericModelKEY=\"$trainatmo\", fObservationModeKEY=\"$trainobs\", fMCParticleKEY=\"$trainpart\", fReflectorVersionKEY=\"$trainrefl\", fCorsikaVersionKEY=\"$traincors\", fCameraVersionKEY=\"$traincam\", fViewConeAngleOKEY=\"$trainview\", fTestTrainKEY=2 where fSequenceFirst=\"$firstruntrain\" " + if [ "$traincheck3" == "" ] + then + mysql -ss -u $user --password=$pw --host=vela $db -e " $traininsert1 " + fi + if [ "$traincheck4" == "" ] + then + mysql -ss -u $user --password=$pw --host=vela $db -e " $traininsert2 " else - mysql -ss -u $user --password=$pw --host=vela $db -e " $update2 " + mysql -ss -u $user --password=$pw --host=vela $db -e " $trainupdate2 " fi - no=`echo $firstrun | cut -c 0-4` - sequpath=$mcsequpath/$no - makedir $sequpath >> $scriptlog 2>&1 - sequfile=$sequpath/sequence$firstrun.txt - echo "writing sequfile "$sequfile >> $scriptlog 2>&1 - printprocesslog "INFO writing sequencefile $sequfile" + trainno=`echo $firstruntrain | cut -c 0-4` + trainsequpath=$mcsequpath/$trainno + makedir $trainsequpath >> $scriptlog 2>&1 + trainsequfile=$trainsequpath/sequence$firstruntrain.txt + echo "writing train sequfile "$trainsequfile >> $scriptlog 2>&1 + printprocesslog "INFO writing train sequencefile $trainsequfile" - echo "Sequence: $firstrun" >| $sequfile - echo "Night: $date" >> $sequfile - echo "Epoch: $epoch" >> $sequfile - echo "Mode: $mode" >> $sequfile - echo "" >> $sequfile - echo "CalRuns: $calrun" >> $sequfile - echo "PedRuns: $pedrun" >> $sequfile - echo "DatRuns: $datruns" >> $sequfile - echo "" >> $sequfile - echo "MonteCarlo: Yes" >> $sequfile - echo "" >> $sequfile + echo "Sequence: $firstruntrain" >| $trainsequfile + echo "Night: $date" >> $trainsequfile + echo "Epoch: $epoch" >> $trainsequfile + echo "Mode: $mode" >> $trainsequfile + echo "" >> $trainsequfile + echo "CalRuns: $calrun" >> $trainsequfile + echo "PedRuns: $pedrun" >> $trainsequfile + echo "DatRuns: $trainruns" >> $trainsequfile + echo "" >> $trainsequfile + echo "MonteCarlo: Yes" >> $trainsequfile + echo "Training sequence" >> $trainsequfile + echo "" >> $trainsequfile + + + #test runs + for testrun in ${testruns[@]} + do + testupdate=" update MCRunData set fSequenceFirst=\"$firstruntest\" where fRunNumber=\"$testrun\" " + mysql -ss -u $user --password=$pw --host=vela $db -e " $testupdate " + done + + testquery1=" select fSpectrumKEY, fPSFKEY, fAmplFadcKEY, fAtmosphericModelKEY, fObservationModeKEY, fMCParticleKEY, fReflectorVersionKEY, fCorsikaVersionKEY, fCameraVersionKEY, fViewConeAngleOKEY, fAddSpotSizeKEY from MCRunData where fRunNumber=\"$firstruntest\" " + testquery2=" select sum(fNumEvents), sum(fNumtriggers) from MCRunData where fRunNumber in ($testruns2) " + testvalues=`mysql -ss -u $user --password=$pw --host=vela $db -e " $testquery1 "` + testvalues2=`mysql -ss -u $user --password=$pw --host=vela $db -e " $testquery2 "` + testspec=`echo $testvalues | cut -d" " -f1` + testpsf=`echo $testvalues | cut -d" " -f2` + testamplfadc=`echo $testvalues | cut -d" " -f3` + testatmo=`echo $testvalues | cut -d" " -f4` + testobs=`echo $testvalues | cut -d" " -f5` + testpart=`echo $testvalues | cut -d" " -f6` + testrefl=`echo $testvalues | cut -d" " -f7` + testcors=`echo $testvalues | cut -d" " -f8` + testcam=`echo $testvalues | cut -d" " -f9` + testview=`echo $testvalues | cut -d" " -f10` + testass=`echo $testvalues | cut -d" " -f11` + testevents=`echo $testvalues2 | cut -d" " -f1` + testtriggers=`echo $testvalues2 | cut -d" " -f2` + testquery3=" select fSequenceFirst from MCSequenceProcessStatus where fSequenceFirst=\"$firstruntest\" " + testquery4=" select fSequenceFirst from MCSequences where fSequenceFirst=\"$firstruntest\" " + testcheck3=`mysql -ss -u $user --password=$pw --host=vela $db -e " $testquery3 "` + testcheck4=`mysql -ss -u $user --password=$pw --host=vela $db -e " $testquery4 "` + testinsert1=" insert MCSequenceProcessStatus set fSequenceFirst=\"$firstruntest\", fSequenceFileWritten=Now() " + testinsert2=" insert MCSequences set fSequenceFirst=\"$firstruntest\", fRunStart=\"$date\", fNumEvents=\"$testevents\", fNumTriggers=\"$testtriggers\", fZBin=\"$zbin\", fSpectrumKEY=\"$testspec\", fPSFKEY=\"$testpsf\", fAddSpotSizeKEY=\"$testass\", fAmplFadcKEY=\"$testamplfadc\", fAtmosphericModelKEY=\"$testatmo\", fObservationModeKEY=\"$testobs\", fMCParticleKEY=\"$testpart\", fReflectorVersionKEY=\"$testrefl\", fCorsikaVersionKEY=\"$testcors\", fCameraVersionKEY=\"$testcam\", fViewConeAngleOKEY=\"$testview\", fTestTrainKEY=3 " + testupdate2=" update MCSequences set fRunStart=\"$date\", fNumEvents=\"$testevents\", fNumTriggers=\"$testtriggers\", fZBin=\"$zbin\", fSpectrumKEY=\"$testspec\", fPSFKEY=\"$testpsf\", fAddSpotSizeKEY=\"$testass\", fAmplFadcKEY=\"$testamplfadc\", fAtmosphericModelKEY=\"$testatmo\", fObservationModeKEY=\"$testobs\", fMCParticleKEY=\"$testpart\", fReflectorVersionKEY=\"$testrefl\", fCorsikaVersionKEY=\"$testcors\", fCameraVersionKEY=\"$testcam\", fViewConeAngleOKEY=\"$testview\", fTestTrainKEY=3 where fSequenceFirst=\"$firstruntest\" " + if [ "$testcheck3" == "" ] + then + mysql -ss -u $user --password=$pw --host=vela $db -e " $testinsert1 " + fi + if [ "$testcheck4" == "" ] + then + mysql -ss -u $user --password=$pw --host=vela $db -e " $testinsert2 " + else + mysql -ss -u $user --password=$pw --host=vela $db -e " $testupdate2 " + fi + + testno=`echo $firstruntest | cut -c 0-4` + testsequpath=$mcsequpath/$testno + makedir $testsequpath >> $scriptlog 2>&1 + testsequfile=$testsequpath/sequence$firstruntest.txt + echo "writing test sequfile "$testsequfile >> $scriptlog 2>&1 + printprocesslog "INFO writing test sequencefile $testsequfile" + + echo "Sequence: $firstruntest" >| $testsequfile + echo "Night: $date" >> $testsequfile + echo "Epoch: $epoch" >> $testsequfile + echo "Mode: $mode" >> $testsequfile + echo "" >> $testsequfile + echo "CalRuns: $calrun" >> $testsequfile + echo "PedRuns: $pedrun" >> $testsequfile + echo "DatRuns: $testruns" >> $testsequfile + echo "" >> $testsequfile + echo "MonteCarlo: Yes" >> $testsequfile + echo "Test sequence" >> $testsequfile + echo "" >> $testsequfile fi done