/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz 11/2005 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2006
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
// MJTrainSeparation
//
////////////////////////////////////////////////////////////////////////////
#include "MJTrainSeparation.h"

#include <TF1.h>
#include <TH2.h>
#include <TChain.h>
#include <TGraph.h>
#include <TMarker.h>
#include <TCanvas.h>
#include <TVirtualPad.h>

#include "MHMatrix.h"

#include "MLog.h"
#include "MLogManip.h"

// tools
#include "MMath.h"
#include "MDataSet.h"
#include "MTFillMatrix.h"
#include "MChisqEval.h"
#include "MStatusDisplay.h"

// eventloop
#include "MParList.h"
#include "MTaskList.h"
#include "MEvtLoop.h"

// tasks
#include "MReadMarsFile.h"
#include "MContinue.h"
#include "MFillH.h"
#include "MRanForestCalc.h"
#include "MParameterCalc.h"

// container
#include "MMcEvt.hxx"
#include "MParameters.h"

// histograms
#include "MBinning.h"
#include "MH3.h"
#include "MHHadronness.h"

// filter
#include "MF.h"
#include "MFEventSelector.h"
#include "MFilterList.h"

ClassImp(MJTrainSeparation);

using namespace std;

void MJTrainSeparation::DisplayResult(MH3 &h31, MH3 &h32)
{
    TH2D &g = (TH2D&)h32.GetHist();
    TH2D &h = (TH2D&)h31.GetHist();

    h.SetMarkerColor(kRed);
    g.SetMarkerColor(kGreen);

    TH2D res1(g);
    TH2D res2(g);

    h.SetTitle("Hadronness-Distribution vs. Size");
    res1.SetTitle("Significance Li/Ma");
    res1.SetXTitle("Size [phe]");
    res1.SetYTitle("Hadronness");
    res2.SetTitle("Significance-Distribution");
    res2.SetXTitle("Size-Cut [phe]");
    res2.SetYTitle("Hadronness-Cut");
    res1.SetContour(50);
    res2.SetContour(50);

    const Int_t nx = h.GetNbinsX();
    const Int_t ny = h.GetNbinsY();

    gROOT->SetSelectedPad(NULL);

    TGraph gr1;
    TGraph gr2;
    for (int x=0; x<nx; x++)
    {
        TH1 *hx = h.ProjectionY("H_py", x+1, x+1);
        TH1 *gx = g.ProjectionY("G_py", x+1, x+1);

        Double_t max1 = -1;
        Double_t max2 = -1;
        Int_t maxy1 = 0;
        Int_t maxy2 = 0;
        for (int y=0; y<ny; y++)
        {
            const Float_t s = gx->Integral(1, y+1);
            const Float_t b = hx->Integral(1, y+1);
            const Float_t sig1 = MMath::SignificanceLiMa(s+b, b);
            const Float_t sig2 = s<1 ? 0 : MMath::SignificanceLiMa(s+b, b)*TMath::Log10(s+1);
            if (sig1>max1)
            {
                maxy1 = y;
                max1 = sig1;
            }
            if (sig2>max2)
            {
                maxy2 = y;
                max2 = sig2;
            }

            res1.SetBinContent(x+1, y+1, sig1);
        }

        gr1.SetPoint(x, h.GetXaxis()->GetBinCenter(x+1), h.GetYaxis()->GetBinCenter(maxy1+1));
        gr2.SetPoint(x, h.GetXaxis()->GetBinCenter(x+1), h.GetYaxis()->GetBinCenter(maxy2+1));

        delete hx;
        delete gx;
    }

    for (int x=0; x<nx; x++)
    {
        TH1 *hx = h.ProjectionY("H_py", x+1);
        TH1 *gx = g.ProjectionY("G_py", x+1);
        for (int y=0; y<ny; y++)
        {
            const Float_t s = gx->Integral(1, y+1);
            const Float_t b = hx->Integral(1, y+1);
            const Float_t sig = MMath::SignificanceLiMa(s+b, b);
            res2.SetBinContent(x+1, y+1, sig);
        }
        delete hx;
        delete gx;
    }

    TGraph gr3;
    TGraph gr4;
    gr4.SetTitle("Significance Li/Ma vs. Hadronness-cut");

    TH1 *hx = h.ProjectionY("H_py");
    TH1 *gx = g.ProjectionY("G_py");
    for (int y=0; y<ny; y++)
    {
        const Float_t s = gx->Integral(1, y+1);
        const Float_t b = hx->Integral(1, y+1);
        const Float_t sig1 = MMath::SignificanceLiMa(s+b, b);
        const Float_t sig2 = s<1 ? 0 : MMath::SignificanceLiMa(s+b, b)*TMath::Log10(s);

        gr3.SetPoint(y, h.GetYaxis()->GetBinLowEdge(y+2), sig1);
        gr4.SetPoint(y, h.GetYaxis()->GetBinLowEdge(y+2), sig2);
    }
    delete hx;
    delete gx;

    TCanvas &c = fDisplay->AddTab("OptCut");
    c.SetBorderMode(0);
    c.Divide(2,2);

    c.cd(1);
    gPad->SetBorderMode(0);
    gPad->SetFrameBorderMode(0);
    gPad->SetLogx();
    gPad->SetGridx();
    gPad->SetGridy();
    h.DrawCopy();
    g.DrawCopy("same");
    gr1.SetMarkerStyle(kFullDotMedium);
    gr1.DrawClone("LP")->SetBit(kCanDelete);
    gr2.SetLineColor(kBlue);
    gr2.SetMarkerStyle(kFullDotMedium);
    gr2.DrawClone("LP")->SetBit(kCanDelete);

    c.cd(3);
    gPad->SetBorderMode(0);
    gPad->SetFrameBorderMode(0);
    gPad->SetGridx();
    gPad->SetGridy();
    gr4.SetMarkerStyle(kFullDotMedium);
    gr4.DrawClone("ALP")->SetBit(kCanDelete);
    gr3.SetLineColor(kBlue);
    gr3.SetMarkerStyle(kFullDotMedium);
    gr3.DrawClone("LP")->SetBit(kCanDelete);

    c.cd(2);
    gPad->SetBorderMode(0);
    gPad->SetFrameBorderMode(0);
    gPad->SetLogx();
    gPad->SetGridx();
    gPad->SetGridy();
    gPad->AddExec("color", "gStyle->SetPalette(1, 0);");
    res1.SetMaximum(7);
    res1.DrawCopy("colz");

    c.cd(4);
    gPad->SetBorderMode(0);
    gPad->SetFrameBorderMode(0);
    gPad->SetLogx();
    gPad->SetGridx();
    gPad->SetGridy();
    gPad->AddExec("color", "gStyle->SetPalette(1, 0);");
    res2.SetMaximum(res2.GetMaximum()*1.05);
    res2.DrawCopy("colz");

    Int_t mx, my, mz;
    res2.GetMaximumBin(mx, my, mz);

    TMarker m;
    m.SetMarkerStyle(kStar);
    m.DrawMarker(res2.GetXaxis()->GetBinCenter(mx), res2.GetYaxis()->GetBinCenter(my));
}

/*
Bool_t  MJSpectrum::InitWeighting(const MDataSet &set, MMcSpectrumWeight &w) const
{
    fLog->Separator("Initialize energy weighting");

    if (!CheckEnv(w))
    {
        *fLog << err << "ERROR - Reading resources for MMcSpectrumWeight failed." << endl;
        return kFALSE;
    }

    TChain chain("RunHeaders");
    set.AddFilesOn(chain);

    MMcCorsikaRunHeader *h=0;
    chain.SetBranchAddress("MMcCorsikaRunHeader.", &h);
    chain.GetEntry(1);

    if (!h)
    {
        *fLog << err << "ERROR - Couldn't read MMcCorsikaRunHeader from DataSet." << endl;
        return kFALSE;
    }

    if (!w.Set(*h))
    {
        *fLog << err << "ERROR - Initializing MMcSpectrumWeight failed." << endl;
        return kFALSE;
    }

    w.Print();
    return kTRUE;
}

Bool_t MJSpectrum::ReadOrigMCDistribution(const MDataSet &set, TH1 &h, MMcSpectrumWeight &weight) const
{
    // Some debug output
    fLog->Separator("Compiling original MC distribution");

    weight.SetNameMcEvt("MMcEvtBasic");
    const TString w(weight.GetFormulaWeights());
    weight.SetNameMcEvt();

    *fLog << inf << "Using weights: " << w << endl;
    *fLog << "Please stand by, this may take a while..." << flush;

    if (fDisplay)
        fDisplay->SetStatusLine1("Compiling MC distribution...");

    // Create chain
    TChain chain("OriginalMC");
    set.AddFilesOn(chain);

    // Prepare histogram
    h.Reset();

    // Fill histogram from chain
    h.SetDirectory(gROOT);
    if (h.InheritsFrom(TH2::Class()))
    {
        h.SetNameTitle("ThetaEMC", "Event-Distribution vs Theta and Energy for MC (produced)");
        h.SetXTitle("\\Theta [\\circ]");
        h.SetYTitle("E [GeV]");
        h.SetZTitle("Counts");
        chain.Draw("MMcEvtBasic.fEnergy:MMcEvtBasic.fTelescopeTheta*TMath::RadToDeg()>>ThetaEMC", w, "goff");
    }
    else
    {
        h.SetNameTitle("ThetaMC", "Event-Distribution vs Theta for MC (produced)");
        h.SetXTitle("\\Theta [\\circ]");
        h.SetYTitle("Counts");
        chain.Draw("MMcEvtBasic.fTelescopeTheta*TMath::RadToDeg()>>ThetaMC", w, "goff");
    }
    h.SetDirectory(0);

    *fLog << "done." << endl;
    if (fDisplay)
        fDisplay->SetStatusLine2("done.");

    if (h.GetEntries()>0)
        return kTRUE;

    *fLog << err << "ERROR - Histogram with original MC distribution empty..." << endl;

    return h.GetEntries()>0;
}
*/

Bool_t MJTrainSeparation::GetEventsProduced(MDataSet &set, Double_t &num, Double_t &min, Double_t &max) const
{
    TChain chain("OriginalMC");
    set.AddFilesOn(chain);

    min = chain.GetMinimum("MMcEvtBasic.fEnergy");
    max = chain.GetMaximum("MMcEvtBasic.fEnergy");

    num = chain.GetEntries();

    if (num<100)
        *fLog << err << "ERROR - Less than 100 entries in OriginalMC-Tree of MC-Train-Data found." << endl;

    return num>=100;
}

Double_t MJTrainSeparation::GetDataRate(MDataSet &set, Double_t &num) const
{
    TChain chain1("Events");
    set.AddFilesOff(chain1);

    num = chain1.GetEntries();
    if (num<100)
    {
        *fLog << err << "ERROR - Less than 100 entries in Events-Tree of Train-Data found." << endl;
        return -1;
    }

    TChain chain("EffectiveOnTime");
    set.AddFilesOff(chain);

    chain.Draw("MEffectiveOnTime.fVal", "MEffectiveOnTime.fVal", "goff");

    TH1 *h = dynamic_cast<TH1*>(gROOT->FindObject("htemp"));
    if (!h)
    {
        *fLog << err << "ERROR - Weird things are happening (htemp not found)!" << endl;
        return -1;
    }

    const Double_t ontime = h->Integral();
    delete h;

    if (ontime<1)
    {
        *fLog << err << "ERROR - Less than 1s of effective observation time found in Train-Data." << endl;
        return -1;
    }

    return num/ontime;
}

Double_t MJTrainSeparation::GetNumMC(MDataSet &set) const
{
    TChain chain1("Events");
    set.AddFilesOn(chain1);

    const Double_t num = chain1.GetEntries();
    if (num<100)
    {
        *fLog << err << "ERROR - Less than 100 entries in Events-Tree of Train-Data found." << endl;
        return -1;
    }

    return num;
}

Bool_t MJTrainSeparation::AutoTrain(MDataSet &set, UInt_t &seton, UInt_t &setoff)
{
    Double_t num, min, max;
    if (!GetEventsProduced(set, num, min, max))
        return kFALSE;

    *fLog << inf << "Using build-in radius of 300m to calculate collection area!" << endl;

    // Target spectrum
    TF1 flx("Flux", "[0]/1000*(x/1000)^(-2.6)", min, max);
    flx.SetParameter(0, 2e-7);

    // Number n0 of events this spectrum would produce per s and m^2
    const Double_t n0 = flx.Integral(min, max);    //[#]

    // Area produced in MC
    const Double_t A = TMath::Pi()*300*300;        //[mē]

    // Rate R of events this spectrum would produce per s
    const Double_t R = n0*A;                       //[Hz]

    *fLog << "Gamma rate from the source inside the MC production area: " << R << "Hz" << endl;

    // Number N of events produced (in trainings sample)
    const Double_t N = num;                        //[#]

    *fLog << "Events produced by MC inside the production area:         " << TMath::Nint(num) << endl;

    // This correponds to an observation time T [s]
    const Double_t T = N/R;                        //[s]

    *fLog << "Total time produced by the Monte Carlo:                   " << T << "s" << endl;

    // With an average data rate after star of
    Double_t data=0;
    const Double_t r = GetDataRate(set, data); //[Hz]

    *fLog << "Events measured per second effective on time:             " << r << "Hz" << endl;
    *fLog << "Total effective on time:                                  " << data/r  << "s" << endl;

    // this yields a number of n events to be read for training
    const Double_t n = r*T;                        //[#]

    *fLog << "Events to be read from the data sample:                   " << TMath::Nint(n) << endl;
    *fLog << "Events available in data sample:                          " << data << endl;

    if (r<0)
        return kFALSE;

    Double_t nummc = GetNumMC(set);

    *fLog << "Events available in MC sample:                            " << nummc << endl;

    *fLog << "MC read probability:                                      " << data/n << endl;

    // more data requested than available => Scale down num MC events
    Double_t on, off;
    if (data<n)
    {
        on  = TMath::Nint(nummc*data/n);
        off = TMath::Nint(data);
        *fLog << warn;
        *fLog << "Not enough data events available... scaling by " << data/n << endl;
        *fLog << inf;
    }
    else
    {
        on  = TMath::Nint(nummc);
        off = TMath::Nint(n);
    }

    if (seton>0 && seton<on)
    {
        setoff = TMath::Nint(off*seton/on);
        *fLog << "Less MC events requested... scaling by " << seton/on << endl;
    }
    else
    {
        seton  = TMath::Nint(on);
        setoff = TMath::Nint(off);
    }

    *fLog << "Target number of MC events:   " << seton  << endl;
    *fLog << "Target number of data events: " << setoff << endl;

    /*
     An event rate dependent selection?
     ----------------------------------
     Total average data rate:      R
     Goal number of events:        N
     Number of data events:        N0
     Rate assigned to single evt:  r

     Selection probability: N/N0 * r/R

     f := N/N0 * r

     MF f("f * MEventRate.fRate < rand");
     */

    return kTRUE;
}

Bool_t MJTrainSeparation::Train(const char *out)
{
    if (!fDataSetTrain.IsValid())
    {
        *fLog << err << "ERROR - DataSet for training invalid!" << endl;
        return kFALSE;
    }
    if (!fDataSetTest.IsValid())
    {
        *fLog << err << "ERROR - DataSet for testing invalid!" << endl;
        return kFALSE;
    }

    // ----------------------- Auto Train? ----------------------

    if (fAutoTrain)
    {
        fLog->Separator("Auto-Training -- Train-Data");
        if (!AutoTrain(fDataSetTrain, fNumTrainOn, fNumTrainOff))
            return kFALSE;
        fLog->Separator("Auto-Training -- Test-Data");
        if (!AutoTrain(fDataSetTest,  fNumTestOn,  fNumTestOff))
            return kFALSE;
    }

    // --------------------- Setup files --------------------
    MReadMarsFile read1("Events");
    MReadMarsFile read2("Events");
    MReadMarsFile read3("Events");
    MReadMarsFile read4("Events");
    read1.DisableAutoScheme();
    read2.DisableAutoScheme();
    read3.DisableAutoScheme();
    read4.DisableAutoScheme();

    fDataSetTrain.AddFilesOn(read1);
    fDataSetTrain.AddFilesOff(read3);

    fDataSetTest.AddFilesOff(read2);
    fDataSetTest.AddFilesOn(read4);

    // ----------------------- Setup RF ----------------------
    MHMatrix train("Train");
    train.AddColumns(fRules);
    train.AddColumn("MHadronness.fVal");

    // ----------------------- Fill Matrix RF ----------------------

    MParameterD had("MHadronness");

    MParList plistx;
    plistx.AddToList(&had);
    plistx.AddToList(this);

    MTFillMatrix fill;
    fill.SetLogStream(fLog);
    fill.SetDisplay(fDisplay);
    fill.AddPreCuts(fPreCuts);
    fill.AddPreCuts(fTrainCuts);

    // Set classifier for gammas
    had.SetVal(0);
    fill.SetName("FillGammas");
    fill.SetDestMatrix1(&train, fNumTrainOn);
    fill.SetReader(&read1);
    if (!fill.Process(plistx))
        return kFALSE;

    const Int_t numgammastrn = train.GetNumRows();
    if (numgammastrn==0)
    {
        *fLog << err << "ERROR - No gammas available for training... aborting." << endl;
        return kFALSE;
    }

    // Set classifier for hadrons
    had.SetVal(1);
    fill.SetName("FillBackground");
    fill.SetDestMatrix1(&train, fNumTrainOff);
    fill.SetReader(&read3);
    if (!fill.Process(plistx))
        return kFALSE;

    const Int_t numbackgrndtrn = train.GetNumRows()-numgammastrn;
    if (numbackgrndtrn==0)
    {
        *fLog << err << "ERROR - No background available for training... aborting." << endl;
        return kFALSE;
    }

    // ------------------------ Train RF --------------------------

    MRanForestCalc rf;
    rf.SetNumTrees(fNumTrees);
    rf.SetNdSize(fNdSize);
    rf.SetNumTry(fNumTry);
    rf.SetNumObsoleteVariables(1);
    rf.SetDebug(fDebug);
    rf.SetDisplay(fDisplay);
    rf.SetLogStream(fLog);
    rf.SetFileName(out);
    rf.SetNameOutput("MHadronness");

    if (fUseRegression)
    {
        if (!rf.TrainSingleRF(train))                  // regression
            return kFALSE;
    }
    else
    {
        MBinning b(2, -0.5, 1.5, "BinningHadronness", "lin");
        if (!rf.TrainSingleRF(train, b.GetEdgesD()))   // classification
            return kFALSE;
    }

    //if (!rf.TrainMultiRF(train, b.GetEdgesD()))    // classification
    //    return;

    //fDisplay = rf.GetDisplay();


    *fLog << all;
    fLog->Separator("The forest was tested with...");

    *fLog << "Training method:" << endl;
    *fLog << " * " << (fUseRegression?"regression":"classification") << endl;
    *fLog << endl;
    *fLog << "Events used for training:"   << endl;
    *fLog << " * Gammas:     " << numgammastrn   << endl;
    *fLog << " * Background: " << numbackgrndtrn << endl;
    *fLog << endl;
    *fLog << "Gamma/Background ratio:" << endl;
    *fLog << " * Requested:  " << (float)fNumTrainOn/fNumTrainOff << endl;
    *fLog << " * Result:     " << (float)numgammastrn/numbackgrndtrn << endl;

    if (!fDataSetTest.IsValid())
        return kTRUE;

    // --------------------- Display result ----------------------
    fLog->Separator("Test");

    MParList  plist;
    MTaskList tlist;
    plist.AddToList(this);
    plist.AddToList(&tlist);

    MMcEvt mcevt;
    plist.AddToList(&mcevt);

    // ----- Setup histograms -----
    MBinning binsy(50, 0 , 1,      "BinningMH3Y", "lin");
    MBinning binsx(40, 10, 100000, "BinningMH3X", "log");

    plist.AddToList(&binsx);
    plist.AddToList(&binsy);

    MH3 h31("MHillas.fSize",  "MHadronness.fVal");
    MH3 h32("MHillas.fSize",  "MHadronness.fVal");
    MH3 h40("MMcEvt.fEnergy", "MHadronness.fVal");
    h31.SetTitle("Background probability vs. Size:Size [phe]:Hadronness");
    h32.SetTitle("Background probability vs. Size:Size [phe]:Hadronness");
    h40.SetTitle("Background probability vs. Energy:Energy [GeV]:Hadronness");

    MHHadronness hist;

    // ----- Setup tasks -----
    MFillH fillh0(&hist, "", "FillHadronness");
    MFillH fillh1(&h31);
    MFillH fillh2(&h32);
    MFillH fillh4(&h40);
    fillh1.SetNameTab("Background");
    fillh2.SetNameTab("GammasH");
    fillh4.SetNameTab("GammasE");
    fillh0.SetBit(MFillH::kDoNotDisplay);

    // ----- Setup filter -----
    MFilterList precuts;
    precuts.AddToList(fPreCuts);
    precuts.AddToList(fTestCuts);

    MContinue c0(&precuts);
    c0.SetName("PreCuts");
    c0.SetInverted();

    MFEventSelector sel;
    sel.SetNumSelectEvts(fNumTestOff);

    MContinue c1(&sel);
    c1.SetInverted();

    // ----- Setup tasklist -----
    tlist.AddToList(&read2);
    tlist.AddToList(&c0);
    tlist.AddToList(&c1);
    tlist.AddToList(&rf);
    tlist.AddToList(&fillh0);
    tlist.AddToList(&fillh1);

    // ----- Run eventloop on gammas -----
    MEvtLoop loop;
    loop.SetDisplay(fDisplay);
    loop.SetLogStream(fLog);
    loop.SetParList(&plist);

    if (!loop.Eventloop())
        return kFALSE;

    // ----- Setup and run eventloop on background -----
    sel.SetNumSelectEvts(fNumTestOn);
    fillh0.ResetBit(MFillH::kDoNotDisplay);

    tlist.Replace(&read4);
    tlist.Replace(&fillh2);
    tlist.AddToListAfter(&fillh4, &fillh2);

    if (!loop.Eventloop())
        return kFALSE;

    DisplayResult(h31, h32);

    if (!WriteDisplay(out))
        return kFALSE;

    *fLog << all;
    fLog->Separator("The forest was tested with...");

    const Double_t numgammastst   = h32.GetHist().GetEntries();
    const Double_t numbackgrndtst = h31.GetHist().GetEntries();

    *fLog << "Events used for test:"   << endl;
    *fLog << " * Gammas:     " << numgammastst   << endl;
    *fLog << " * Background: " << numbackgrndtst << endl;
    *fLog << endl;
    *fLog << "Gamma/Background ratio:" << endl;
    *fLog << " * Requested:  " << (float)fNumTestOn/fNumTestOff << endl;
    *fLog << " * Result:     " << (float)numgammastst/numbackgrndtst << endl;

    return kTRUE;
}

