/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Markus Gaug   02/2004 <mailto:markus@ifae.es>
!              
!   Copyright: MAGIC Software Development, 2000-2004
!
!
\* ======================================================================== */
/////////////////////////////////////////////////////////////////////////////
//                                                                        
// MHCalibrationHiLoCam                                                
//                                                                        
// Fills the extracted high-gain low-gain charge ratios of MArrivalTimeCam into 
// the MHCalibrationPix-classes MHCalibrationPix for every:
//
// - Pixel, stored in the TObjArray's MHCalibrationCam::fHiGainArray  
//   or MHCalibrationCam::fHiGainArray, respectively, depending if 
//   MArrivalTimePix::IsLoGainUsed() is set.
//
// - Average pixel per AREA index (e.g. inner and outer for the MAGIC camera), 
//   stored in the TObjArray's MHCalibrationCam::fAverageHiGainAreas and 
//   MHCalibrationCam::fAverageHiGainAreas
//
// - Average pixel per camera SECTOR (e.g. sectors 1-6 for the MAGIC camera), 
//   stored in the TObjArray's MHCalibrationCam::fAverageHiGainSectors 
//   and MHCalibrationCam::fAverageHiGainSectors 
//
// The histograms are fitted to a Gaussian, mean and sigma with its errors 
// and the fit probability are extracted. If none of these values are NaN's and 
// if the probability is bigger than MHGausEvents::fProbLimit (default: 0.5%), 
// the fit is declared valid.
// Otherwise, the fit is repeated within ranges of the previous mean 
// +- MHCalibrationPix::fPickupLimit (default: 5) sigma (see MHCalibrationPix::RepeatFit())
// In case this does not make the fit valid, the histogram means and RMS's are 
// taken directly (see MHCalibrationPix::BypassFit()) and the following flags are set:
// - MBadPixelsPix::SetUncalibrated( MBadPixelsPix::kHiLoNotFitted ) and
// - MBadPixelsPix::SetUnsuitable(   MBadPixelsPix::kUnreliableRun    ) 
// 
// Outliers of more than MHCalibrationPix::fPickupLimit (default: 5) sigmas 
// from the mean are counted as Pickup events (stored in MHCalibrationPix::fPickup) 
//
// The class also fills arrays with the signal vs. event number, creates a fourier 
// spectrum (see MHGausEvents::CreateFourierSpectrum()) and investigates if the 
// projected fourier components follow an exponential distribution. 
// In case that the probability of the exponential fit is less than 
// MHGausEvents::fProbLimit (default: 0.5%), the following flags are set:
// - MBadPixelsPix::SetUncalibrated( MBadPixelsPix::kHiLoOscillating ) and
// - MBadPixelsPix::SetUnsuitable(   MBadPixelsPix::kUnreliableRun      )
// 
// This same procedure is performed for the average pixels.
//
// The following results are written into MCalibrationHiLoCam:
//
// - MCalibrationPix::SetMean()
// - MCalibrationPix::SetMeanErr()
// - MCalibrationPix::SetSigma()
// - MCalibrationPix::SetSigmaErr()
// - MCalibrationPix::SetProb()
// - MCalibrationPix::SetNumPickup()
//
// For all averaged areas, the fitted sigma is multiplied with the square root of 
// the number involved pixels in order to be able to compare it to the average of 
// sigmas in the camera.
//                                                                         
/////////////////////////////////////////////////////////////////////////////
#include "MHCalibrationHiLoCam.h"
#include "MHCalibrationPix.h"

#include "MLog.h"
#include "MLogManip.h"

#include "MParList.h"

#include "MCalibrationIntensityHiLoCam.h"

#include "MCalibrationHiLoCam.h"
#include "MCalibrationCam.h"
#include "MCalibrationPix.h"

#include "MExtractedSignalCam.h"
#include "MExtractedSignalPix.h"

#include "MRawEvtPixelIter.h"

#include "MGeomCam.h"
#include "MGeomPix.h"

#include "MBadPixelsIntensityCam.h"
#include "MBadPixelsCam.h"
#include "MBadPixelsPix.h"

#include <TOrdCollection.h>
#include <TPad.h>
#include <TVirtualPad.h>
#include <TCanvas.h>
#include <TStyle.h>
#include <TF1.h>
#include <TLine.h>
#include <TLatex.h>
#include <TLegend.h>
#include <TGraph.h>

ClassImp(MHCalibrationHiLoCam);

using namespace std;

const Int_t   MHCalibrationHiLoCam::fgNbins    = 200;
const Axis_t  MHCalibrationHiLoCam::fgFirst    = 0.;
const Axis_t  MHCalibrationHiLoCam::fgLast     = 20.;
const Float_t MHCalibrationHiLoCam::fgProbLimit = 0.;
const TString MHCalibrationHiLoCam::gsHistName  = "HiLo";
const TString MHCalibrationHiLoCam::gsHistTitle = "HiGain vs. LoGain";
const TString MHCalibrationHiLoCam::gsHistXTitle = "Amplification Ratio [1]";
const TString MHCalibrationHiLoCam::gsHistYTitle = "Nr. events";
const Byte_t  MHCalibrationHiLoCam::fgLowerLim   = 200;
const Byte_t  MHCalibrationHiLoCam::fgUpperLim   = 252;
// --------------------------------------------------------------------------
//
// Default Constructor.
//
// Sets: 
// - fNbins to fgNbins
// - fFirst to fgFirst
// - fLast  to fgLast 
//
// - fHistName   to gsHistName  
// - fHistTitle  to gsHistTitle 
// - fHistXTitle to gsHistXTitle
// - fHistYTitle to gsHistYTitle
//
// - fLowerLimt  to fgLowerLim
// - fUpperLimt  to fgUpperLim
//
MHCalibrationHiLoCam::MHCalibrationHiLoCam(const char *name, const char *title) 
{

  fName  = name  ? name  : "MHCalibrationHiLoCam";
  fTitle = title ? title : "Histogram class for the high-gain vs. low-gain amplification ratio calibration";

  SetNbins(fgNbins);
  SetFirst(fgFirst);
  SetLast (fgLast );

  SetProbLimit(fgProbLimit);

  SetHistName  (gsHistName  .Data());
  SetHistTitle (gsHistTitle .Data());
  SetHistXTitle(gsHistXTitle.Data());
  SetHistYTitle(gsHistYTitle.Data());

  SetLowerLim();
  SetUpperLim();
}

// --------------------------------------------------------------------------
//
// Creates new MHCalibrationHiLoCam only with the averaged areas:
// the rest has to be retrieved directly, e.g. via: 
//     MHCalibrationHiLoCam *cam = MParList::FindObject("MHCalibrationHiLoCam");
//  -  cam->GetAverageSector(5).DrawClone();
//  -  (*cam)[100].DrawClone()
//
TObject *MHCalibrationHiLoCam::Clone(const char *) const
{

  MHCalibrationHiLoCam *cam = new MHCalibrationHiLoCam();

  //
  // Copy the data members
  //
  cam->fColor                  = fColor;
  cam->fRunNumbers             = fRunNumbers;
  cam->fPulserFrequency        = fPulserFrequency;
  cam->fFlags                  = fFlags;
  cam->fNbins                  = fNbins;
  cam->fFirst                  = fFirst;
  cam->fLast                   = fLast;

  //
  // Copy the MArrays
  //
  cam->fAverageAreaRelSigma    = fAverageAreaRelSigma;
  cam->fAverageAreaRelSigmaVar = fAverageAreaRelSigmaVar;
  cam->fAverageAreaSat         = fAverageAreaSat;
  cam->fAverageAreaSigma       = fAverageAreaSigma;
  cam->fAverageAreaSigmaVar    = fAverageAreaSigmaVar;
  cam->fAverageAreaNum         = fAverageAreaNum;
  cam->fAverageSectorNum       = fAverageSectorNum;

  if (!IsAverageing())
    return cam;

  const Int_t navhi   =  fAverageHiGainAreas->GetSize();

  for (int i=0; i<navhi; i++)
    cam->fAverageHiGainAreas->AddAt(GetAverageHiGainArea(i).Clone(),i);

  return cam;
}

// --------------------------------------------------------------------------
//
// Gets the pointers to:
// - MRawEvtData
//
Bool_t MHCalibrationHiLoCam::SetupHists(const MParList *pList)
{

  fRawEvt = (MRawEvtData*)pList->FindObject("MRawEvtData");
  if (!fRawEvt)
  {
      *fLog << err << dbginf << "MRawEvtData not found... aborting." << endl;
      return kFALSE;
  }

  return kTRUE;
}

// --------------------------------------------------------------------------
//
// Gets or creates the pointers to:
// - MCalibrationHiLoCam
//
// Searches pointer to:
// - MExtractedSignalCam
//
// Calls:
// - MHCalibrationCam::InitHiGainArrays()
// 
// Sets:
// - SetLoGain(kFALSE);
// - fSumarea   to nareas
// - fSumsector to nareas
// - fNumarea   to nareas
// - fNumsector to nareas
//
Bool_t MHCalibrationHiLoCam::ReInitHists(MParList *pList)
{

  fIntensCam = (MCalibrationIntensityCam*)pList->FindObject(AddSerialNumber("MCalibrationIntensityHiLoCam"));
  if (fIntensCam)
    *fLog << inf << "Found MCalibrationIntensityHiLoCam ... " << endl;
  else
    {
      fCam = (MCalibrationCam*)pList->FindObject(AddSerialNumber("MCalibrationHiLoCam"));
      if (!fCam)
        {
          fCam = (MCalibrationCam*)pList->FindCreateObj(AddSerialNumber("MCalibrationHiLoCam"));
          if (!fCam)
            {
              *fLog << err << "Cannot find nor create MCalibrationHiLoCam ... abort." << endl;
              return kFALSE;
            }
          fCam->Init(*fGeom);
        }
    }

  MExtractedSignalCam *signal = (MExtractedSignalCam*)pList->FindObject("MExtractedSignalCam");
  if (!signal)
  {
      *fLog << err << "MExtractedSignalCam not found... abort." << endl;
      return kFALSE;
  }

  const Int_t npixels  = fGeom->GetNumPixels();
  const Int_t nsectors = fGeom->GetNumSectors();
  const Int_t nareas   = fGeom->GetNumAreas();

  InitHiGainArrays(npixels,nareas,nsectors);

  fSumarea  .Set(nareas); 
  fSumsector.Set(nsectors); 
  fNumarea  .Set(nareas); 
  fNumsector.Set(nsectors); 

  SetLoGain(kFALSE);

  return kTRUE;
}

// -------------------------------------------------------------------------------
//
// Retrieves pointer to MExtractedSignalCam:
//
// Retrieves from MGeomCam:
// - number of pixels
// - number of pixel areas
// - number of sectors
//
// Fills histograms (MHGausEvents::FillHistAndArray()) with:
// - MExtractedSignalPix::GetExtractedSignalHiGain(pixid) / MExtractedSignalPix::GetExtractedSignalLoGain;
//   if the high-gain signal lies in between the limits: fLowerLim and fUpperLim
//
Bool_t MHCalibrationHiLoCam::FillHists(const MParContainer *par, const Stat_t w)
{

  MExtractedSignalCam *signal = (MExtractedSignalCam*)par;
  if (!signal)
    {
      gLog << err << "No argument in MExtractedSignal::Fill... abort." << endl;
      return kFALSE;
    }
  
  const Int_t nareas   = fGeom->GetNumAreas();
  const Int_t nsectors = fGeom->GetNumSectors();

  fSumarea  .Reset();
  fSumsector.Reset(); 
  fNumarea  .Reset(); 
  fNumsector.Reset(); 

  MRawEvtPixelIter pixel(fRawEvt);

  while (pixel.Next())
    {
      
      const Byte_t max = pixel.GetMaxHiGainSample();
      
      if (max < fLowerLim || max > fUpperLim)
       continue;

      const UInt_t pixidx = pixel.GetPixelId();

      MHCalibrationPix &hist = (*this)[pixidx];

      if (hist.IsExcluded())
	continue;

      const MExtractedSignalPix &pix = (*signal)[pixidx];
      const Int_t aidx   = (*fGeom)[pixidx].GetAidx();
      const Int_t sector = (*fGeom)[pixidx].GetSector();

      const Float_t ratio = pix.GetExtractedSignalHiGain() / pix.GetExtractedSignalLoGain();
      
      hist.FillHistAndArray(ratio) ;
      fSumarea  [aidx]   += ratio;
      fNumarea  [aidx]   ++;
      fSumsector[sector] += ratio;
      fNumsector[sector] ++;
    }
  
  for (Int_t j=0; j<nareas; j++)
    {
      MHCalibrationPix &hist = GetAverageHiGainArea(j);
      hist.FillHistAndArray(fNumarea[j] == 0 ? 0. : fSumarea[j]/fNumarea[j]);
    }
  
  for (Int_t j=0; j<nsectors; j++)
    {
      MHCalibrationPix &hist = GetAverageHiGainSector(j);
      hist.FillHistAndArray(fNumsector[j] == 0 ? 0. : fSumsector[j]/fNumsector[j]);
    }

  return kTRUE;
}

// --------------------------------------------------------------------------
//
// Calls:
// - MHCalibrationCam::FitHiGainArrays() with flags:
//   MBadPixelsPix::kHiLoNotFitted and MBadPixelsPix::kHiLoOscillating
// - MHCalibrationCam::FitLoGainArrays() with flags:
//   MBadPixelsPix::kHiLoNotFitted and MBadPixelsPix::kHiLoOscillating
// 
Bool_t MHCalibrationHiLoCam::FinalizeHists()
{

  *fLog << endl;

  MCalibrationCam *hilocam = fIntensCam ? fIntensCam->GetCam() : fCam;
  MBadPixelsCam   *badcam = fIntensBad ? fIntensBad->GetCam() : fBadPixels;

  const Int_t nareas = fAverageHiGainAreas->GetSize();
  const Int_t nsectors = fAverageHiGainSectors->GetSize();

  TArrayI satarea(nareas);
  TArrayI satsect(nsectors);
  fNumarea  .Reset(); 
  fNumsector.Reset(); 

  for (Int_t i=0; i<fHiGainArray->GetSize(); i++)
    {
      
      MHCalibrationPix &hist = (*this)[i];

      if (hist.IsExcluded())
        continue;
      
      const Int_t aidx   = (*fGeom)[i].GetAidx();
      const Int_t sector = (*fGeom)[i].GetSector();

      fNumarea[aidx]++;
      fNumsector[sector]++;
      //
      // Check histogram overflow
      // 
      CheckOverflow(hist);
    }

  for (Int_t j=0; j<nareas; j++)
    {
      
      MHCalibrationPix    &hist = GetAverageHiGainArea(j);      
      //
      // Check histogram overflow
      // 
      CheckOverflow(hist);
   }
  
  for (Int_t j=0; j<fAverageHiGainSectors->GetSize(); j++)
    {
      
      MHCalibrationPix &hist = GetAverageHiGainSector(j);      
      //
      // Check histogram overflow
      // 
      CheckOverflow(hist);
    }

  FitHiGainArrays(*hilocam,*badcam,
                  MBadPixelsPix::kHiLoNotFitted,
                  MBadPixelsPix::kHiLoOscillating);
  
  return kTRUE;
}

// --------------------------------------------------------------------------
//
// Sets all pixels to MBadPixelsPix::kUnreliableRun, if following flags are set:
// - MBadPixelsPix::kHiLoNotFitted
// - MBadPixelsPix::kHiLoOscillating
//
void MHCalibrationHiLoCam::FinalizeBadPixels()
{

  MBadPixelsCam *badcam = fIntensBad ? fIntensBad->GetCam() : fBadPixels;

  for (Int_t i=0; i<badcam->GetSize(); i++)
    {
      MBadPixelsPix          &bad    = (*badcam)[i];

      if (bad.IsUncalibrated( MBadPixelsPix::kHiLoNotFitted ))
        bad.SetUnsuitable(   MBadPixelsPix::kUnreliableRun    );

      if (bad.IsUncalibrated( MBadPixelsPix::kHiLoOscillating))
        bad.SetUnsuitable(   MBadPixelsPix::kUnreliableRun    );
      
    }
}

// --------------------------------------------------------------------------
//
// The types are as follows:
// 
// Fitted values:
// ============== 
//
// 0: Fitted Mean High-Gain Low-Gain Charge Ratio in FADC slices  (MHGausEvents::GetMean()
// 1: Error Mean High-Gain Low-Gain Charge Ratio in FADC slices   (MHGausEvents::GetMeanErr()
// 2: Sigma fitted High-Gain Low-Gain Charge Ratio in FADC slices (MHGausEvents::GetSigma()
// 3: Error Sigma High-Gain Low-Gain Charge Ratio in FADC slices  (MHGausEvents::GetSigmaErr()
//
// Useful variables derived from the fit results:
// =============================================
//
// 4: Returned probability of Gauss fit              (calls: MHGausEvents::GetProb())
//
// Localized defects:
// ==================
//
// 5: Gaus fit not OK                               (calls: MHGausEvents::IsGausFitOK())
// 6: Fourier spectrum not OK                       (calls: MHGausEvents::IsFourierSpectrumOK())
//
Bool_t MHCalibrationHiLoCam::GetPixelContent(Double_t &val, Int_t idx, const MGeomCam &cam, Int_t type) const
{

  if (fHiGainArray->GetSize() <= idx)
    return kFALSE;

  const MHCalibrationPix &pix = (*this)[idx];

  switch (type)
    {
    case 0:
      val = pix.GetMean();
      break;
    case 1:
      val = pix.GetMeanErr();
      break;
    case 2:
      val = pix.GetSigma();
      break;
    case 3:
      val = pix.GetSigmaErr();
      break;
    case 4:
      val = pix.GetProb();
      break;
    case 5:
      if (!pix.IsGausFitOK())
        val = 1.;
      break;
    case 6:
      if (!pix.IsFourierSpectrumOK())
        val = 1.;
      break;
    default:
      return kFALSE;
    }
  return kTRUE;
}

// --------------------------------------------------------------------------
//
// Calls MHCalibrationPix::DrawClone() for pixel idx
//
void MHCalibrationHiLoCam::DrawPixelContent(Int_t idx) const
{
 (*this)[idx].DrawClone();
}

void MHCalibrationHiLoCam::CheckOverflow( MHCalibrationPix &pix )
{

  if (pix.IsExcluded())
    return;

  TH1F *hist = pix.GetHGausHist();
  
  Stat_t overflow = hist->GetBinContent(hist->GetNbinsX()+1);
  if (overflow > 0.0005*hist->GetEntries())
    {
      *fLog << warn << "Hist-overflow " << overflow
            << " times in " << pix.GetName() << " (w/o saturation!) " << endl;
    }
  
  overflow = hist->GetBinContent(0);
  if (overflow > 0.0005*hist->GetEntries())
    {
      *fLog << warn << "Hist-underflow " << overflow
            << " times in " << pix.GetName() << " (w/o saturation!) " << endl;
    }
}