source: trunk/MagicSoft/Mars/mbadpixels/MBadPixelsTreat.cc@ 6634

/* ======================================================================== *\
!
! *
! * This file is part of MARS, the MAGIC Analysis and Reconstruction
! * Software. It is distributed to you in the hope that it can be a useful
! * and timesaving tool in analysing Data of imaging Cerenkov telescopes.
! * It is distributed WITHOUT ANY WARRANTY.
! *
! * Permission to use, copy, modify and distribute this software and its
! * documentation for any purpose is hereby granted without fee,
! * provided that the above copyright notice appear in all copies and
! * that both that copyright notice and this permission notice appear
! * in supporting documentation. It is provided "as is" without express
! * or implied warranty.
! *
!
!
!   Author(s): Oscar Blanch 12/2001 <mailto:blanch@ifae.es>
!   Author(s): Thomas Bretz 08/2002 <mailto:tbretz@astro.uni-wuerzburg.de>
!
!   Copyright: MAGIC Software Development, 2000-2004
!
!
\* ======================================================================== */

/////////////////////////////////////////////////////////////////////////////
//
//  MBadPixelsTreat
//
//  You can use MBadPixelsTreat::SetUseInterpolation to replaced the
//  bad pixels by the average of its neighbors instead of unmapping
//  them. If you want to include the central pixel use
//  MBadPixelsTreat::SetUseCentralPixel. The bad pixels are taken from
//  an existing MBadPixelsCam.
//
//  It check if there are enough neighbors to calculate the mean
//  If not, unmap the pixel. The minimum number of good neighbors
//  should be set using SetNumMinNeighbors
//
//  If you want to interpolate unreliable pixels and  unsuitable
//  (broken) pixels use SetHardTreatment().
//
//
// Options:
// --------
//   SetHardTreatment:      Also interpolate unreliable pixels not only unsuitable
//   SetUseInterpolation:   Interpolate pixels instead of unmapping them
//   SetUseCentralPixel:    also use the pixel itself for interpolation
//   SetProcessPedestalRun: process the pedestals once per run/file
//   SetProcessPedestalEvt: process the pedestal once per event
//   SetProcessTimes:       also do some rough interpolation of the arrival time
//
//
//  Input Containers:
//   MCerPhotEvt
//   MPedPhotCam
//   MBadPixelsCam
//   [MGeomCam]
//
//  Output Containers:
//   MCerPhotEvt
//
/////////////////////////////////////////////////////////////////////////////
#include "MBadPixelsTreat.h"

#include <fstream>

#include <TEnv.h>
#include <TObjString.h>

#include "MArrayD.h" // Used instead of TArrayD because operator[] has no range check

#include "MLog.h"
#include "MLogManip.h"

#include "MParList.h"

#include "MGeomPix.h"
#include "MGeomCam.h"

#include "MCerPhotPix.h"
#include "MCerPhotEvt.h"

#include "MPedPhotPix.h"
#include "MPedPhotCam.h"

#include "MBadPixelsPix.h"
#include "MBadPixelsCam.h"

#include "MArrivalTime.h"

ClassImp(MBadPixelsTreat);

using namespace std;

static const TString gsDefName  = "MBadPixelsTreat";
static const TString gsDefTitle = "Task to treat bad pixels (interpolation, unmapping)";

// --------------------------------------------------------------------------
//
// Default constructor.
//
MBadPixelsTreat::MBadPixelsTreat(const char *name, const char *title)
  : fFlags(0), fNumMinNeighbors(3)
{
    fName  = name  ? name  : gsDefName.Data();
    fTitle = title ? title : gsDefTitle.Data();

    SetUseInterpolation();
    SetProcessPedestal();
    SetProcessTimes();
}

// --------------------------------------------------------------------------
//
// Returns the status of a pixel. If kHardTreatment is set a pixel must
// be unsuitable or uriliable to be treated. If not it is treated only if 
// it is unsuitable
// (IsBad() checks for any flag)
//
Bool_t MBadPixelsTreat::IsPixelBad(Int_t idx) const
{
    return TESTBIT(fFlags, kHardTreatment) ? (*fBadPixels)[idx].IsBad():(*fBadPixels)[idx].IsUnsuitable();
}

void MBadPixelsTreat::AddNamePedPhotCam(const char *name)
{
    fNamePedPhotCams.Add(new TObjString(name));
}

// --------------------------------------------------------------------------
//
//  - Try to find or create MBlindPixels in parameter list.
//  - get the MCerPhotEvt from the parlist (abort if missing)
//  - if no pixels are given by the user try to determin the starfield
//    from the monte carlo run header.
//
Int_t MBadPixelsTreat::PreProcess (MParList *pList)
{
    fBadPixels = (MBadPixelsCam*)pList->FindObject(AddSerialNumber("MBadPixelsCam"));
    if (!fBadPixels)
    {
        *fLog << err << AddSerialNumber("MBadPixelsCam") << " not found... aborting." << endl;
        return kFALSE;
    }

    fEvt = (MCerPhotEvt*)pList->FindObject(AddSerialNumber("MCerPhotEvt"));
    if (!fEvt)
    {
        *fLog << err << AddSerialNumber("MCerPhotEvt") << " not found... aborting." << endl;
        return kFALSE;
    }

    fGeomCam = 0;
    if (!IsUseInterpolation())
        return kTRUE;

    fGeomCam = (MGeomCam*)pList->FindObject(AddSerialNumber("MGeomCam"));
    if (!fGeomCam)
    {
        *fLog << err << AddSerialNumber("MGeomCam") << " not found - can't use interpolation... abort." << endl;
        *fLog << " Use MBadPixelsTreat::SetUseInterpolation(kFALSE) to switch interpolation" << endl;
        *fLog << " off and use unmapping instead." << endl;
        return kFALSE;
    }

    const Bool_t proc = IsProcessPedestalEvt() || IsProcessPedestalRun();

    if (fNamePedPhotCams.GetSize()>0 && !proc)
    {
        *fLog << err << "Pedestal list contains entries, but pedestal treatment is switched off... abort." << endl;
        return kFALSE;
    }

    if (proc)
    {
        if (fNamePedPhotCams.GetSize()==0)
        {
            *fLog << inf << "No container names specified... using default: MPedPhotCam." << endl;
            AddNamePedPhotCam();
        }

        fPedPhotCams.Clear();

        TIter Next(&fNamePedPhotCams);
        TObject *o=0;
        while ((o=Next()))
        {
            TObject *p = pList->FindObject(AddSerialNumber(o->GetName()), "MPedPhotCam");
            if (!p)
            {
                *fLog << err << AddSerialNumber(o->GetName()) << " [MPedPhotCam] not found... aborting." << endl;
                //*fLog << " Use  MBadPixelsTreat::SetProcessPedestalRun(kFALSE)  and" << endl;
                //*fLog << " MBadPixelsTreat::SetProcessPedestalEvt(kFALSE) to switch" << endl;
                //*fLog << " Pedestal treatment off." << endl;
                return kFALSE;
            }

            fPedPhotCams.Add(p);
        }
    }

    fTimes = 0;
    if (IsProcessTimes())
    {
        fTimes = (MArrivalTime*)pList->FindObject(AddSerialNumber("MArrivalTime"));
        if (!fTimes)
        {
            *fLog << err << AddSerialNumber("MArrivalTime") << " not found... aborting." << endl;
            *fLog << " Use MBadPixelsTreat::SetProcessTimes(kFALSE) to switch time interpolation off." << endl;
            return kFALSE;
        }
    }

    if (IsProcessPedestalEvt())
        *fLog << inf << "Processing Pedestals once per event..." << endl;
    if (IsProcessPedestalRun())
        *fLog << inf << "Processing Pedestals once per run..." << endl;
    if (IsProcessTimes())
        *fLog << inf << "Processing Arrival Times once per event..." << endl;

    return kTRUE;
}

// --------------------------------------------------------------------------
//
//  Replaces each pixel (signal, signal error, pedestal, pedestal rms)
//  by the average of its surrounding pixels.
//  If TESTBIT(fFlags, kUseCentralPixel) is set the central pixel is also
//  included.
//
void MBadPixelsTreat::InterpolateSignal() const
{
    const UShort_t entries = fGeomCam->GetNumPixels();

    //
    // Create arrays (FIXME: Check if its possible to create it only once)
    //
    MArrayD nphot(entries);
    MArrayD perr(entries);
 
    //
    // Loop over all pixels
    //
    for (UShort_t i=0; i<entries; i++)
    {
        MCerPhotPix *pix = fEvt->GetPixById(i);

        //
        // Check whether pixel with idx i is blind
        //
        if (pix && !IsPixelBad(i))
            continue;

        //
        // Get a pointer to this pixel. If it is not yet existing
        // create a new entry for this pixel in MCerPhotEvt
        //
        if (!pix)
        {
            pix = fEvt->AddPixel(i, 0, 0);
            (*fBadPixels)[i].SetUnsuitable(MBadPixelsPix::kUnsuitableEvt);
        }

        //
        // Get the corresponding geometry and pedestal
        //
        const MGeomPix &gpix = (*fGeomCam)[i];

        // Do Not-Use-Central-Pixel
        const Bool_t nucp = !TESTBIT(fFlags, kUseCentralPixel);

        Int_t num = nucp ? 0 : 1;

        nphot[i]  = nucp ? 0 : pix->GetNumPhotons();
        perr[i]   = nucp ? 0 : Pow2(pix->GetErrorPhot());

        //
        // The values are rescaled to the small pixels area for the right comparison
        //
        const Double_t ratio = fGeomCam->GetPixRatio(i);

        nphot[i] *= ratio;
        perr[i]  *= ratio;

        //
        // Loop over all its neighbors
        //
        const Int_t n = gpix.GetNumNeighbors();
        for (int j=0; j<n; j++)
        {
            const UShort_t nidx = gpix.GetNeighbor(j);

            //
            // Do not use blind neighbors
            //
            if (IsPixelBad(nidx))
                continue;

            //
            // Check whether the neighbor has a signal stored
            //
            const MCerPhotPix *evtpix = fEvt->GetPixById(nidx);
            if (!evtpix)
                continue;

            //
            // Get the geometry for the neighbor
            //
            const Double_t nratio = fGeomCam->GetPixRatio(nidx);

            //
            //The error is calculated as the quadratic sum of the errors
            //
            nphot[i] += nratio*evtpix->GetNumPhotons();
            perr[i]  += nratio* Pow2(evtpix->GetErrorPhot());

            num++;
        }

        // Check if there are enough neighbors to calculate the mean
        // If not, unmap the pixel. The maximum number of blind neighbors
        // should be 2
        if (num<fNumMinNeighbors)
        {
            pix->SetPixelUnmapped();
            continue;
        }

        //
        // Now the mean is calculated and the values rescaled back
        // to the pixel area
        //
        nphot[i] /= (num*ratio);
        perr[i]   = TMath::Sqrt(perr[i]/(num*ratio));
 
        pix->Set(nphot[i], perr[i]);
    }
}

// --------------------------------------------------------------------------
//
void MBadPixelsTreat::InterpolatePedestals(MPedPhotCam &pedphot) const
{
    const Int_t entries = pedphot.GetSize();

    // Create arrays (FIXME: Check if its possible to create it only once)
    MArrayD ped(entries);
    MArrayD rms(entries);

    //
    // Loop over all pixels
    //
    for (UShort_t i=0; i<entries; i++)
    {
        //
        // Check whether pixel with idx i is blind
        //
        if (!IsPixelBad(i))
            continue;

        //
        // Get the corresponding geometry and pedestal
        //
        const MGeomPix    &gpix = (*fGeomCam)[i];
        const MPedPhotPix &ppix = pedphot[i];

        // Do Not-Use-Central-Pixel
        const Bool_t nucp = !TESTBIT(fFlags, kUseCentralPixel);

        Int_t num = nucp ? 0 : 1;

        ped[i] = nucp ? 0 : ppix.GetMean();
        rms[i] = nucp ? 0 : Pow2(ppix.GetRms());

        //
        // The values are rescaled to the small pixels area for the right comparison
        //
        const Double_t ratio = fGeomCam->GetPixRatio(i);

        ped[i] *= ratio;
        rms[i] *= ratio;

        //
        // Loop over all its neighbors
        //
        const Int_t n = gpix.GetNumNeighbors();
        for (int j=0; j<n; j++)
        {
            const UShort_t nidx = gpix.GetNeighbor(j);

            //
            // Do not use blind neighbors
            //
            if (IsPixelBad(nidx))
                continue;

            //
            // Get the geometry for the neighbor
            //
            const Double_t    nratio = fGeomCam->GetPixRatio(nidx);
            const MPedPhotPix &nppix = pedphot[nidx];

            //
            //The error is calculated as the quadratic sum of the errors
            //
            ped[i] += nratio*nppix.GetMean();
            rms[i] += nratio*Pow2(nppix.GetRms());

            num++;
        }

        // Check if there are enough neighbors to calculate the mean
        // If not, unmap the pixel. The minimum number of good neighbors
        // should be fNumMinNeighbors
        if (num < fNumMinNeighbors)
        {
            MCerPhotPix *pix =fEvt->GetPixById(i);
            if (!pix)
                pix = fEvt->AddPixel(i, 0, 0);
            pix->SetPixelUnmapped();
            continue;
        }

        //
        // Now the mean is calculated and the values rescaled back
        // to the pixel area
        //
        ped[i] /=  (num*ratio);
        rms[i]  = TMath::Sqrt(rms[i]/(num*ratio));

        pedphot[i].Set(ped[i], rms[i]);
    }
    pedphot.SetReadyToSave();
}

// --------------------------------------------------------------------------
//
// loop over all MPedPhotCam and interpolate them
//
void MBadPixelsTreat::InterpolatePedestals() const
{
    TIter Next(&fPedPhotCams);
    MPedPhotCam *cam=0;
    while ((cam=(MPedPhotCam*)Next()))
    {
        InterpolatePedestals(*cam);
        cam->ReCalc(*fGeomCam, fBadPixels);
    }
}

// --------------------------------------------------------------------------
//
void MBadPixelsTreat::InterpolateTimes() const
{
    Int_t n = fTimes->GetSize();

    for (int i=0; i<n; i++)
    {
        //
        // Check whether pixel with idx i is blind
        //
        if (!IsPixelBad(i))
            continue;

        //
        // Get the corresponding geometry and pedestal
        //
        const MGeomPix &gpix = (*fGeomCam)[i];

        MArrayD time(gpix.GetNumNeighbors());

        Int_t n0 = 0;
        for (unsigned int j=0; j<time.GetSize(); j++)
        {
            const Double_t t = (*fTimes)[gpix.GetNeighbor(j)];
            if (t>=0)
                time[n0++] = t;
        }

        Int_t p0=-1;
        Int_t p1=-1;

        Double_t min=FLT_MAX;
        for (int j=0; j<n0; j++)
            for (int k=0; k<j; k++)
            {
                const Double_t diff = TMath::Abs(time[j] - time[k]);

                if (diff>=min && diff<250)
                    continue;

                p0  = j;
                p1  = k;
                min = diff;
            }

        // FIXME: Request a minimum number of neighbors to take action?
        if (p0>=0 && p1>=0 && TMath::Abs(time[p0] - time[p1])<250)
            fTimes->SetTime(i, (time[p0]+time[p1])/2);
    }
}

// --------------------------------------------------------------------------
//
//  Replaces each pixel (signal, signal error, pedestal, pedestal rms)
//  by the average of its surrounding pixels.
//  If TESTBIT(fFlags, kUseCentralPixel) is set the central pixel is also
//  included.
//
//  NT: Informations about the interpolated pedestals are added. 
//      When the option Interpolated is called, the blind pixel with the new
//      values of signal and fluttuation is included in the calculation of
//      the Image Parameters.
//
/*
void MBadPixelsTreat::Interpolate() const
{
    const UShort_t entries = fGeomCam->GetNumPixels();

    //
    // Create arrays
    //
    TArrayD nphot(entries);
    TArrayD perr(entries);
    TArrayD ped(entries);
    TArrayD pedrms(entries);
 
    //
    // Loop over all pixels
    //
    for (UShort_t i=0; i<entries; i++)
    {
        MCerPhotPix *pix = fEvt->GetPixById(i);

        //
        // Check whether pixel with idx i is blind
        //
        if (pix && (*fBadPixels)[i].IsOK())
            continue;

        //
        // Get a pointer to this pixel. If it is not yet existing
        // create a new entry for this pixel in MCerPhotEvt
        //
        if (!pix)
        {
            pix = fEvt->AddPixel(i, 0, 0);
            (*fBadPixels)[i].SetUnsuitable(MBadPixelsPix::kUnsuitableEvt);
        }

        //
        // Get the corresponding geometry and pedestal
        //
        const MGeomPix &gpix    = (*fGeomCam)[i];
        const MPedPhotPix &ppix = (*fPedPhot)[i];

        // Do Not-Use-Central-Pixel
        const Bool_t nucp = !TESTBIT(fFlags, kUseCentralPixel);

        Int_t num = nucp ? 0 : 1;

        nphot[i]  = nucp ? 0 : pix->GetNumPhotons();
        ped[i]    = nucp ? 0 : ppix.GetMean();
        perr[i]   = nucp ? 0 : Pow2(pix->GetErrorPhot());
        pedrms[i] = nucp ? 0 : Pow2(ppix.GetRms());

        //
        // The values are rescaled to the small pixels area for the right comparison
        //
        const Double_t ratio = fGeomCam->GetPixRatio(i);

        if (nucp)
        {
            nphot[i]  *= ratio;
            perr[i]   *= ratio;
            ped[i]    *= ratio;
            pedrms[i] *= ratio;
        }

        //
        // Loop over all its neighbors
        //
        const Int_t n = gpix.GetNumNeighbors();
        for (int j=0; j<n; j++)
        {
            const UShort_t nidx = gpix.GetNeighbor(j);

            //
            // Do not use blind neighbors
            //
            if ((*fBadPixels)[nidx].IsBad())
                continue;

            //
            // Check whether the neighbor has a signal stored
            //
            const MCerPhotPix *evtpix = fEvt->GetPixById(nidx);
            if (!evtpix)
                continue;

            //
            // Get the geometry for the neighbor
            //
            const Double_t nratio = fGeomCam->GetPixRatio(nidx);
            MPedPhotPix &nppix    = (*fPedPhot)[nidx];

            //
            // The error is calculated as the quadratic sum of the errors
            //
            nphot[i]  += nratio*evtpix->GetNumPhotons();
            ped[i]    += nratio*nppix.GetMean();
            perr[i]   += nratio*Pow2(evtpix->GetErrorPhot());
            pedrms[i] += nratio*Pow2(nppix.GetRms());

            num++;
        }

        if (num<fNumMinNeighbors)
        {
            pix->SetPixelUnmapped();
            nphot[i]  = 0;
            ped[i]    = 0;
            perr[i]   = 0;
            pedrms[i] = 0;
            continue;
        }

        //
        // Now the mean is calculated and the values rescaled back to the pixel area
        //
        nphot[i] /= num*ratio;
        ped[i]   /= num*ratio;
        perr[i]   = TMath::Sqrt(perr[i]/(num*ratio));
        pedrms[i] = TMath::Sqrt(pedrms[i]/(num*ratio));

    }

    //
    // Now the new pixel values are calculated and can be replaced in
    // the corresponding containers
    //
    for (UShort_t i=0; i<entries; i++)
    {
        //
        // Do not use blind neighbors
        //
        if ((*fBadPixels)[i].IsOK())
            continue;

        //
        // It must exist, we have created it in the loop before.
        //
        fEvt->GetPixById(i)->Set(nphot[i], perr[i]);
        (*fPedPhot)[i].Set(ped[i], pedrms[i]);
    }
}
*/

// --------------------------------------------------------------------------
//
//  Removes all blind pixels from the analysis by setting their state
//  to unused.
//
void MBadPixelsTreat::Unmap() const
{
    const UShort_t entries = fEvt->GetNumPixels();

    //
    // remove the pixels in fPixelsIdx if they are set to be used,
    // (set them to 'unused' state)
    //
    for (UShort_t i=0; i<entries; i++)
    {
        MCerPhotPix &pix = (*fEvt)[i];

        if (IsPixelBad(pix.GetPixId()))
            pix.SetPixelUnmapped();
    }
}

// --------------------------------------------------------------------------
//
// Interpolate Pedestals if kProcessPedestal not set
//
Bool_t MBadPixelsTreat::ReInit(MParList *pList)
{
    if (IsUseInterpolation() && IsProcessPedestalRun())
        InterpolatePedestals();
    return kTRUE;
}

// --------------------------------------------------------------------------
//
// Treat the blind pixels
//
Int_t MBadPixelsTreat::Process()
{
    if (IsUseInterpolation())
    {
        InterpolateSignal();
        if (IsProcessPedestalEvt())
            InterpolatePedestals();
        if (IsProcessTimes())
            InterpolateTimes();
    }
    else
        Unmap();

    return kTRUE;
}

void MBadPixelsTreat::StreamPrimitive(ofstream &out) const
{
    out << "   MBadPixelsTreat " << GetUniqueName();
    if (fName!=gsDefName || fTitle!=gsDefTitle)
    {
        out << "(\"" << fName << "\"";
        if (fTitle!=gsDefTitle)
            out << ", \"" << fTitle << "\"";
        out <<")";
    }
    out << ";" << endl;

    if (IsUseInterpolation())
        out << "   " << GetUniqueName() << ".SetUseInterpolation();" << endl;
    if (IsUseCentralPixel())
        out << "   " << GetUniqueName() << ".SetUseCentralPixel();" << endl;
    if (IsProcessPedestalRun())
        out << "   " << GetUniqueName() << ".SetProcessPedestalRun();" << endl;
    if (IsProcessPedestalEvt())
        out << "   " << GetUniqueName() << ".SetProcessPedestalEvt();" << endl;
    if (IsProcessTimes())
        out << "   " << GetUniqueName() << ".SetProcessTimes();" << endl;
    if (IsHardTreatment())
        out << "   " << GetUniqueName() << ".SetHardTreatment();" << endl;
    if (fNumMinNeighbors!=3)
        out << "   " << GetUniqueName() << ".SetNumMinNeighbors(" << (int)fNumMinNeighbors << ");" << endl;
}

// --------------------------------------------------------------------------
//
// Read the setup from a TEnv, eg:
//   MBadPixelsTreat.UseInterpolation:    no
//   MBadPixelsTreat.UseCentralPixel:     no
//   MBadPixelsTreat.HardTreatment:       no
//   MBadPixelsTreat.ProcessPedestalRun:  no
//   MBadPixelsTreat.ProcessPedestalEvt:  no
//   MBadPixelsTreat.NumMinNeighbors:     3
//
Int_t MBadPixelsTreat::ReadEnv(const TEnv &env, TString prefix, Bool_t print)
{
    Bool_t rc = kFALSE;
    if (IsEnvDefined(env, prefix, "UseInterpolation", print))
    {
        rc = kTRUE;
        SetUseInterpolation(GetEnvValue(env, prefix, "UseInterpolation", IsUseInterpolation()));
    }
    if (IsEnvDefined(env, prefix, "UseCentralPixel", print))
    {
        rc = kTRUE;
        SetUseCentralPixel(GetEnvValue(env, prefix, "UseCentralPixel", IsUseCentralPixel()));
    }
    if (IsEnvDefined(env, prefix, "HardTreatment", print))
    {
        rc = kTRUE;
        SetHardTreatment(GetEnvValue(env, prefix, "HardTreatment", IsHardTreatment()));
    }
    if (IsEnvDefined(env, prefix, "ProcessPedestalRun", print))
    {
        rc = kTRUE;
        SetProcessPedestalRun(GetEnvValue(env, prefix, "ProcessPedestalRun", IsProcessPedestalRun()));
    }
    if (IsEnvDefined(env, prefix, "ProcessPedestalEvt", print))
    {
        rc = kTRUE;
        SetProcessPedestalEvt(GetEnvValue(env, prefix, "ProcessPedestalEvt", IsProcessPedestalEvt()));
    }
    if (IsEnvDefined(env, prefix, "ProcessTimes", print))
    {
        rc = kTRUE;
        SetProcessTimes(GetEnvValue(env, prefix, "ProcessTimes", IsProcessTimes()));
    }
    if (IsEnvDefined(env, prefix, "NumMinNeighbors", print))
    {
        rc = kTRUE;
        SetNumMinNeighbors(GetEnvValue(env, prefix, "NumMinNeighbors", fNumMinNeighbors));
    }
    return rc;
}