/* ======================================================================== *\
! $Name: not supported by cvs2svn $:$Id: plotrundb.C,v 1.4 2007-10-07 21:12:15 tbretz Exp $
! --------------------------------------------------------------------------
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Thomas Bretz, 05/2005 <mailto:tbretz@astro.uni-wuerzburg.de>
!   Author(s): Daniela Dorner, 05/2005 <mailto:dorner@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2000-2006
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
// plotundb.C
// ==========
//
// This macro is used to read quality parameters from the DB and plot them.
// 
// The parameters are from the following files:
// calib*.root:mean conversion factor, mean arrival time, rms arrival time
// (each parameter for inner and outer camera)
// signal*.root: mean pedestal rms (for inner and outer camera)
// star*.root: PSF, # of Muons, Effective OnTime, Muon rate,
// Ratio MC/Data(MuonSize) and mean number of islands
// 
// In the DB these values are stored in the tables Calibration and Star.
// 
// Usage:
//   .x plotundb.C   --> all values in the DB are plotted
// You can chose are certain period:
//   .x plotrundb.C(25)   --> all values from period 25 are plotted
// or a time period from a certain date to a certain date
//   .x plotrundb.C("2004-11-14 00:00:00", "2005-02-28 00:00:00")
//  --> all values from 14.11.2004 0h to 28.2.2005 0h are plotted
// or all data, but with dataset data highlighted
//   .x plotrundb.C("dataset.txt")
//  --> the sequences defined in dataset.txt are highlighted (blue:on, red:off)
//  --> You can also add a dataset-name as last argument to one of the
//      calls above
//
// Make sure, that database and password are corretly set in a resource
// file called sql.rc and the resource file is found.
//
/////////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <iomanip>

#include <TH1.h>
#include <TEnv.h>
#include <TPad.h>
#include <TLine.h>
#include <TText.h>
#include <TFrame.h>
#include <TStyle.h>
#include <TCanvas.h>
#include <TPRegexp.h>
#include <TSQLRow.h>
#include <TSQLResult.h>
#include <TGraphErrors.h>

#include "MTime.h"
#include "MAstro.h"
#include "MDataSet.h"
#include "MSQLMagic.h"
#include "MStatusDisplay.h"

class MPlot : public MParContainer
{
public:
    // Possible constants to group-by (average) over a certain period
    enum GroupBy_t
    {
        kNone,
        kGroupByPrimary,
        kGroupByHour,
        kGroupByNight,
        kGroupByWeek,
        kGroupByMonth,
        kGroupBySeason,
        kGroupByYear
    };

private:
    MSQLMagic &fServer;         // Reference to the sql-server class

    MDataSet *fDataSet;         // A possible dtaset to highlite single points

    TString   fPrimaryDate;     // The name of the data we plot
    TString   fPrimaryNumber;   // The corresponding name for the key number
    TString   fSecondary;       // The value versus which the second plot is made

    TString   fRequestFrom;     // Start of a requested date range
    TString   fRequestTo;       // End of a requested date range
    Int_t     fRequestPeriod;   // A possible requested period

    Float_t   fPlotMin;
    Float_t   fPlotMax;

    Float_t   fHistMin;
    Float_t   fHistMax;

    TString   fDescription;     // The description (title) of the plot
    TString   fNameTab;         // The name of the tab in the display

    TString   fCondition;       // An additional condition added to the query
    GroupBy_t fGroupBy;         // A possible Group-By flag

    // --------------------------------------------------------------------------
    //
    // Function to plot the result of the query
    //
    void PlotTable(TSQLResult &res, TString name, Float_t fmin, Float_t fmax, Float_t resolution)
    {
        // Enable all otions in the statistics box
        gStyle->SetOptStat(111111);

        // Create TGraph objects
        TGraph &gt = res.GetFieldCount()>4 ? *new TGraphErrors : *new TGraph;
        gt.SetNameTitle(name, Form("%s vs Time", name.Data()));
        gt.SetMarkerStyle(kFullDotMedium);

        TGraph gz;
        gz.SetNameTitle(name, Form("%s vs <Zd>", name.Data()));
        gz.SetMarkerStyle(kFullDotMedium);

        TGraph gt0, gt1;
        gt0.SetMarkerColor(kRed);
        gt1.SetMarkerColor(kBlue);
        gt0.SetMarkerStyle(kFullDotLarge);
        gt1.SetMarkerStyle(kFullDotLarge);

        TGraph gz0, gz1;
        gz0.SetMarkerColor(kRed);
        gz1.SetMarkerColor(kBlue);
        gz0.SetMarkerStyle(kFullDotLarge);
        gz1.SetMarkerStyle(kFullDotLarge);

        Int_t first = -1;
        Int_t last  = -1;

        // Loop over the data
        TSQLRow *row=0;
        while ((row=res.Next()))
        {
            // Get all fields of this row
            const char *date = (*row)[0];
            const char *zd   = (*row)[1];
            const char *val  = (*row)[2];
            const char *snum = res.GetFieldCount()>3 ? (*row)[3] : 0;
            const char *verr = res.GetFieldCount()>4 ? (*row)[5] : 0;

            delete row;

            if (!date || !val || !zd)
                continue;

            // check if date is valid
            MTime t(date);
            if (!t.SetSqlDateTime(date))
                continue;

            // check if it belongs to the requested MAGIC period
            if (fRequestPeriod>0 && MAstro::GetMagicPeriod(t.GetMjd())!=fRequestPeriod)
                continue;

            // Get axis range
            if (first<0)
                first = TMath::Nint(TMath::Floor(t.GetMjd()));
            last = TMath::Nint(TMath::Ceil(t.GetMjd()));

            // Convert a possible key number into a integer
            UInt_t seq = snum ? atoi(snum) : 0;

            // convert primary and secondary value into floats
            Float_t value = atof(val);
            Float_t zenith = atof(zd);

            // If a datset is given add the point to the special TGraphs
            // used for highliting these dates
            if (fDataSet)
            {
                if (fDataSet->HasOnSequence(seq))
                {
                    gt1.SetPoint(gt1.GetN(), t.GetAxisTime(), value);
                    gz1.SetPoint(gz1.GetN(), zenith, value);
                }

                if (fDataSet->HasOffSequence(seq))
                {
                    gt0.SetPoint(gt0.GetN(), t.GetAxisTime(), value);
                    gz0.SetPoint(gz0.GetN(), zenith, value);
                }
            }

            // Add Data to TGraph
            gt.SetPoint(gt.GetN(), t.GetAxisTime(), value);
            gz.SetPoint(gz.GetN(), zenith, value);

            // Set error-bar, if one
            if (verr)
                static_cast<TGraphErrors&>(gt).SetPointError(gt.GetN()-1, 0, atof(verr));
        }

        // If this is done earlier the plots remain empty since root 5.12/00
        if (fmax>fmin)
        {
            gt.SetMinimum(fmin);
            gt.SetMaximum(fmax);
            gz.SetMinimum(fmin);
            gz.SetMaximum(fmax);
        }

        gROOT->SetSelectedPad(0);

        // Create a TCanvas or open a new tab
        TString title = fNameTab.IsNull() ? name(name.First('.')+2, name.Length()) : fNameTab;
        TCanvas &c = fDisplay ? fDisplay->AddTab(title) : *new TCanvas;
        // Set fillcolor, remove border and divide pad
        c.SetFillColor(kWhite);
        c.SetBorderMode(0);
        c.Divide(1,2);

        // Output mean and rms to console
        cerr << setprecision(4) << setw(10) << title << ":   ";
        if (gt.GetN()==0)
        {
            cerr << "     <empty>" << endl;
            return;
        }
        cerr << setw(8) << gt.GetMean(2) << "+-" << setw(8) << gt.GetRMS(2) << "   ";
        if (gt0.GetN()>0 || gt1.GetN()>0)
        {
            cerr << setw(8) << gt1.GetMean(2) << "+-" << setw(8) << gt1.GetRMS(2) << "   ";
            cerr << setw(8) << gt0.GetMean(2) << "+-" << setw(8) << gt0.GetRMS(2);
        }
        cerr << endl;

        TVirtualPad *pad = gPad;

        // draw contants of pad 2 (counting starts at 0)
        pad->cd(2);
        gPad->SetBorderMode(0);
        gPad->SetFrameBorderMode(0);
        gPad->SetGridy();

        gPad->SetLeftMargin(0.06);
        gPad->SetRightMargin(0.06);
        gPad->SetBottomMargin(0.08);

        // format axis
        TH1 *h = gt.GetHistogram();

        h->SetXTitle("Time");
        h->SetYTitle(name);
        h->GetXaxis()->SetTimeDisplay(1);
        h->GetYaxis()->SetTitleOffset(0.8);
        h->GetXaxis()->SetTitleOffset(1.0);
        h->GetXaxis()->SetLabelOffset(0.01);

        // draw TGraph
        gt.DrawClone("AP");
        if (gt0.GetN()>0)
            gt0.DrawClone("P");
        if (gt1.GetN()>0)
            gt1.DrawClone("P");

        // Add lines and text showing the MAGIC periods
        TLine l;
        TText t;
        Int_t num=0;
        l.SetLineStyle(kDotted);
        l.SetLineColor(kBlue);
        t.SetTextColor(kBlue);
        l.SetLineWidth(1);
        t.SetTextSize(h->GetXaxis()->GetLabelSize());
        t.SetTextAlign(21);
        Int_t p0 = MAstro::GetMagicPeriod(first);
        for (Int_t p = first; p<last; p++)
        {
            Int_t p1 = MAstro::GetMagicPeriod(p);
            if (p1!=p0)
            {
                l.DrawLine(MTime(p).GetAxisTime(), h->GetMinimum(), MTime(p).GetAxisTime(), h->GetMaximum());
                t.DrawText(MTime(p+15).GetAxisTime(), h->GetMaximum(), Form("%d", p1));
                num++;
            }
            p0 = p1;
        }
        if (num<4)
            gPad->SetGridx();

        const Double_t min = fHistMin>fHistMax ? h->GetMinimum()-resolution/2 : fHistMin;
        const Double_t max = fHistMin>fHistMax ? h->GetMaximum()+resolution/2 : fHistMax;

        // Use this to save the pad with the time development to a file
        //gPad->SaveAs(Form("plotdb-%s.eps", title.Data()));

        // Go back to first (upper) pad, format it and divide it again
        pad->cd(1);
        gPad->SetBorderMode(0);
        gPad->SetFrameBorderMode(0);
        gPad->Divide(2,1);

        TVirtualPad *pad2 = gPad;

        // format left pad
        pad2->cd(1);
        gPad->SetBorderMode(0);
        gPad->SetFrameBorderMode(0);
        gPad->SetGridx();
        gPad->SetGridy();

        // Create histogram
        const Int_t n = resolution>0 ? TMath::Nint((max-min)/resolution) : 50;

        TH1F hist("Hist", Form("Distribution of %s", fDescription.IsNull() ? name.Data() : fDescription.Data()), n, min, max);
        hist.SetDirectory(0);

        // Fill data into histogra,
        for (int i=0; i<gt.GetN(); i++)
            hist.Fill(gt.GetY()[i]);

        // Format histogram
        if (fDescription.IsNull())
            hist.SetXTitle(name);
        hist.SetYTitle("Counts");

        // plot histogram
        hist.DrawCopy("");

        // format right pad
        pad2->cd(2);
        gPad->SetBorderMode(0);
        gPad->SetFrameBorderMode(0);
        gPad->SetGridy();

        // format graph
        TH1 *h2 = gz.GetHistogram();

        h2->SetXTitle("Zd");
        h2->SetYTitle(name);

        // draw graph
        gz.DrawClone("AP");

        if (gz0.GetN()>0)
            gz0.DrawClone("P");
        if (gz1.GetN()>0)
            gz1.DrawClone("P");
    }

public:
    MPlot(MSQLMagic &server) : fServer(server), fDataSet(NULL),
        fRequestPeriod(-1), fPlotMin(0), fPlotMax(-1), fHistMin(0), fHistMax(-1), fGroupBy(kNone)
    {
    }
    ~MPlot()
    {
        if (fDataSet)
            delete fDataSet;
    }
    void SetDataSet(const TString filename)
    {
        if (fDataSet)
        {
            delete fDataSet;
            fDataSet = NULL;
        }
        if (!filename.IsNull())
            fDataSet = new MDataSet(filename);
    }
    void SetPlotRange(Float_t min, Float_t max, Int_t n=5) { fPlotMin = min; fPlotMax = max; }
    void SetHistRange(Float_t min, Float_t max) { fHistMin = min; fHistMax = max; }
    void SetRequestRange(const char *from="", const char *to="") { fRequestFrom = from; fRequestTo = to; }
    void SetRequestPeriod(Int_t n=-1) { fRequestPeriod = n; }
    void SetCondition(const char *cond="") { fCondition = cond; }
    void SetDescription(const char *d, const char *t=0) { fDescription = d; fNameTab = t; }
    void SetGroupBy(GroupBy_t b=kGroupByWeek) { fGroupBy=b; }
    void SetPrimaryDate(const char *ts) { fPrimaryDate=ts; }
    void SetPrimaryNumber(const char *ts) { fPrimaryNumber=ts; }
    void SetSecondary(const char *ts) { fSecondary=ts; }

    Bool_t Plot(const char *value, Float_t min=0, Float_t max=-1, Float_t resolution=0)
    {
        TString named  = fPrimaryDate;
        TString named2 = fSecondary;
        TString namev  = value;

        TString tablev = namev(0, namev.First('.'));
        TString valuev = namev(namev.First('.')+1, namev.Length());

        TString tabled = named(0, named.First('.'));
        TString valued = named(named.First('.')+1, named.Length());

        TString query="SELECT ";
        switch (fGroupBy)
        {
        case kNone:
        case kGroupByPrimary:
            query += Form("%s AS %s", valued.Data(), valued.Data()+1);
            break;
        case kGroupByHour:
            query += Form("DATE_FORMAT(%s, '%%Y-%%m-%%d %%H:30:00') AS %s ", fPrimaryDate.Data(), valued.Data()+1);
            break;
        case kGroupByNight:
            query += Form("DATE_FORMAT(ADDDATE(%s,Interval 12 hour), '%%Y-%%m-%%d 00:00:00') AS %s ", fPrimaryDate.Data(), valued.Data()+1);
            break;
        case kGroupByWeek:
            query += Form("DATE_FORMAT(ADDDATE(%s,Interval 12 hour), '%%x%%v') AS %s ", fPrimaryDate.Data(), valued.Data()+1);
            break;
        case kGroupByMonth:
            query += Form("DATE_FORMAT(ADDDATE(%s,Interval 12 hour), '%%Y-%%m-15 00:00:00') AS %s ", fPrimaryDate.Data(), valued.Data()+1);
            break;
        case kGroupBySeason:
            //query += Form("DATE_FORMAT(ADDDATE(%s,Interval 12 hour), '%%Y-%%m-15 00:00:00') AS %s ", fPrimaryDate.Data(), valued.Data()+1);
            break;
        case kGroupByYear:
            query += Form("DATE_FORMAT(ADDDATE(%s,Interval 12 hour), '%%Y-08-15 00:00:00') AS %s ", fPrimaryDate.Data(), valued.Data()+1);
            break;
        }

        if (fGroupBy==kNone)
        {
            query += ", ";
            query += fSecondary;
            query += ", ";
            query += value;
            query += ", ";
            query += fPrimaryNumber;
            query += " ";
        }
        else
        {
            query += ", AVG(";
            query += fSecondary;
            query += "), AVG(";
            query += value;
            query += "), ";
            query += fPrimaryNumber;
            query += ", STD(";
            query += fSecondary;
            query += "), STD(";
            query += value;
            query += ") ";
        }

        query += Form("FROM %s ", tabled.Data());

        const Bool_t interval = !fRequestFrom.IsNull() && !fRequestTo.IsNull();

        TString where(fCondition);
        if (!where.IsNull())
            where += " AND ";
        where += "RunData.fExcludedFDAKEY=1  ";

        if (interval)
        {
            if (!where.IsNull())
                where += " AND ";
            where += Form("%s BETWEEN '%s' AND '%s' ",
                          fPrimaryDate.Data(), fRequestFrom.Data(), fRequestTo.Data());
        }

        // ------------------------------

        query += fServer.GetJoins(tabled, query+" "+where);

        if (!where.IsNull())
        {
            query += "WHERE ";
            query += where;
        }

        if (fGroupBy!=kNone)
        {
            query += Form("GROUP BY %s ", valued.Data()+1);
            //query += Form(" HAVING COUNT(%s)=(COUNT(*)+1)/2 ", valuev.Data());
        }
        query += Form("ORDER BY %s ", valued.Data()+1);


        // ------------------------------

        TSQLResult *res = fServer.Query(query);
        if (!res)
        {
            cout << "ERROR - Query failed: " << query << endl;
            return kFALSE;
        }

        if (max>min)
            PlotTable(*res, namev, min, max, resolution);
        else
            PlotTable(*res, namev, fPlotMin, fPlotMax, resolution);


        delete res;
        return kTRUE;
    }
};

void plotall(MPlot &plot)
{
    plot.SetGroupBy(MPlot::kGroupByNight);

    plot.SetPrimaryDate("RunData.fRunStart");
    plot.SetPrimaryNumber("RunData.fRunNumber");
    plot.SetSecondary("RunData.fZenithDistance");

    //inner camera
    //from calib*.root
    plot.SetDescription("DAQ Storage Rate;R_{DAQ,S} [Hz]", "StoreRate");
    plot.Plot("RunData.fDaqStoreRate",  -0.5, 999.5, 5);
    plot.SetDescription("DAQ Trigger Rate;R_{DAQ,T} [Hz]", "TrigRate");
    plot.Plot("RunData.fDaqTriggerRate", -0.5, 999.5, 5);
    plot.SetDescription("Mean Trigger Rate;<R> [Hz]", "MeanTrig");
    plot.Plot("RunData.fMeanTriggerRate", -0.5, 999.5, 5);
    plot.SetDescription("L2 Trigger rate after prescaler;R_{L2,P}", "Presc");
    plot.Plot("RunData.fL2RatePresc", -0.5, 999.5, 5);
    plot.SetDescription("L2 Trigger rate before prescaler;R_{L2,U}", "Unpresc");
    plot.Plot("RunData.fL2RateUnpresc", -0.5, 999.5, 5);

    //from signal*.root
    plot.SetDescription("Signal Position;<T_{S}> [sl]", "PosMean");
    plot.Plot("DataCheck.fPositionSignal",  -0.5, 29.5, 1);
    plot.SetDescription("Signal Position FWHM;\\sigma_{S} [sl]", "PosFWHM");
    plot.Plot("DataCheck.fPositionFWHM",  -0.5, 29.5, 1);
    plot.SetDescription("Signal Height;<H_{s}> [counts]", "SigMean");
    plot.Plot("DataCheck.fHeightSignal",  -0.5, 256.5, 1);
    plot.SetDescription("Signal Height FWHM;\\sigma_{H} [counts]", "SigFWHM");
    plot.Plot("DataCheck.fHeightFWHM",  -0.5, 155.5, 1);

    plot.SetDescription("Interleaved Signal Position;<T_{S}> [sl]", "IPosMean");
    plot.Plot("DataCheck.fPositionSignalInterlaced", -0.5, 29.5, 1);
    plot.SetDescription("Interleaved Signal Position FWHM;\\sigma_{S} [sl]", "IPosFWHM");
    plot.Plot("DataCheck.fPositionFWHMInterlaced", -0.5, 29.5, 1);
    plot.SetDescription("Interleaved Signal Height;<H_{s}> [counts]", "ISigMean");
    plot.Plot("DataCheck.fHeightSignalInterlaced", -0.5, 256.5, 1);
    plot.SetDescription("Interleaved Signal Height FWHM;\\sigma_{H} [counts]", "ISigFWHM");
    plot.Plot("DataCheck.fHeightFWHMInterlaced",  -0.5, 155.5, 1);
}

int plotrundb(TString from, TString to, const char *dataset=0)
{
    TEnv env("sql.rc");

    MSQLMagic serv(env);
    if (!serv.IsConnected())
    {
        cout << "ERROR - Connection to database failed." << endl;
        return 0;
    }

    cout << "plotrundb" << endl;
    cout << "---------" << endl;
    cout << endl;
    cout << "Connected to " << serv.GetName() << endl;
    cout << endl;

    MStatusDisplay *d = new MStatusDisplay;
    d->SetWindowName(serv.GetName());
    d->SetTitle(serv.GetName());

    MPlot plot(serv);
    plot.SetDataSet(dataset);
    plot.SetDisplay(d);
    plot.SetRequestRange(from, to);
    plotall(plot);
    d->SaveAsRoot("plotrundb.root");
    d->SaveAsPS("plotrundb.ps");

    return 1;
}

int plotrundb(const char *ds)
{
    TEnv env("sql.rc");

    MSQLMagic serv(env);
    if (!serv.IsConnected())
    {
        cout << "ERROR - Connection to database failed." << endl;
        return 0;
    }

    cout << "plotrundb" << endl;
    cout << "---------" << endl;
    cout << endl;
    cout << "Connected to " << serv.GetName() << endl;
    cout << endl;

    MStatusDisplay *d = new MStatusDisplay;
    d->SetWindowName(serv.GetName());
    d->SetTitle(serv.GetName());

    MPlot plot(serv);
    plot.SetDataSet(ds);
    plot.SetDisplay(d);
    plot.SetRequestRange("", "");
    plotall(plot);
    d->SaveAsRoot("plotrundb.root");
    d->SaveAsPS("plotrundb.ps");

    return 1;
}

int plotrundb(Int_t period, const char *dataset="")
{
    TEnv env("sql.rc");

    MSQLMagic serv(env);
    if (!serv.IsConnected())
    {
        cout << "ERROR - Connection to database failed." << endl;
        return 0;
    }

    cout << "plotrundb" << endl;
    cout << "---------" << endl;
    cout << endl;
    cout << "Connected to " << serv.GetName() << endl;
    cout << endl;

    MStatusDisplay *d = new MStatusDisplay;
    d->SetWindowName(serv.GetName());
    d->SetTitle(serv.GetName());

    MPlot plot(serv);
    plot.SetDataSet(dataset);
    plot.SetDisplay(d);
    plot.SetRequestPeriod(period);
    plotall(plot);
    d->SaveAsRoot("plotrundb.root");
    d->SaveAsPS("plotrundb.ps");

    return 1;
}

int plotrundb()
{
    return plotrundb("", "");
}