source: trunk/FACT++/src/Fits.cc@ 11705

// **************************************************************************
/** @class FactFits

@brief FITS writter for the FACT project. 

The FactFits class is able to open, manage and update FITS files. 

The file columns should be given to the class before the file is openned. Once
a file has been created, the structure of its columns cannot be changed. Only
row can be added.

This class relies on the CCfits and CFitsIO packages.

*/
// **************************************************************************
#include "Fits.h"

#include "Time.h"
#include "Converter.h"
#include "MessageImp.h"


#include <sys/stat.h> //for file stats

using namespace std;
using namespace CCfits;

// --------------------------------------------------------------------------
//
//! This gives a standard variable to the file writter. 
//! This variable should not be related to the DIM service being logged. 
//! @param desc the description of the variable to add
//! @param dataFormat the FITS data format corresponding to the variable to add.
//! @param dataPointer the memory location where the variable is stored
//! @param numDataBytes the number of bytes taken by the variable
//
void Fits::AddStandardColumn(const Description& desc, const string &dataFormat, void* dataPointer, long unsigned int numDataBytes)
{
        //check if entry already exist
        for (vector<Description>::const_iterator it=fStandardColDesc.begin(); it != fStandardColDesc.end(); it++)
                if (it->name == desc.name)
                        return;
        fStandardColDesc.push_back(desc);
        fStandardFormats.push_back(dataFormat);
        fStandardPointers.push_back(dataPointer);
        fStandardNumBytes.push_back(numDataBytes);
}
// --------------------------------------------------------------------------
//
//! This gives the file writter access to the DIM data
//! @param desc a vector containing the description of all the columns to log
//! @param dataFormat a vector containing the FITS data format of all the columsn to log
//! @param dataPointer the memory location where the DIM data starts
//! @param numDataBytes the number of bytes taken by the DIM data. 
//      
void Fits::InitDataColumns(const vector<Description> &desc, vector<string>& dataFormat, void* dataPointer, int numDataBytes)
{//we will copy this information here. It duplicates the data, which is not great, but it is the easiest way of doing it right now
        if (desc.size() == dataFormat.size())
        {
                fDataColDesc = desc;
        }
        else
        {
                fDataColDesc.clear();
                for (unsigned int i=0;i<dataFormat.size();i++)
                {
                        ostringstream stt;
                        stt << "Data" << i;
                        fDataColDesc.push_back(Description(stt.str(), "comment", "unit"));
                }
        }               
        fDataFormats = dataFormat;
        fDataPointer = dataPointer;
        fDataNumBytes = numDataBytes;   
}
// --------------------------------------------------------------------------
//
//! This looks for a suitable table in the fits file, i.e. that corresponds to the name and column names. (no format check yet)
//! @param tableName. the base table name to be obtained. If not suitable, numbers are appened to the name
//! @param allNames. the name of all columns
//! @param allDataTypes. the data types of all columns
//! @param allUnits. the units of the columns
//! @return a pointer to the newly retrieved/created table
//
CCfits::Table* Fits::findSuitableTableInFitsFile(const string& tableName,const vector<string>& allNames, const vector<string>& allDataTypes, const vector<string>& allUnits)
{
    const multimap< string, CCfits::ExtHDU * >& extMap = fFile->extension();
    for (int i=0;i<100;i++)
    {
        if (i==10)
            fMess->Error("Already 10 different tables with different formats exist in this file. Please consider re-creating the file entirely (i.e. delete it please)");
        ostringstream cTableName;
        cTableName << tableName;
        if (i != 0)
            cTableName << "-" << i;
        //current table name does not exist yet. return its associated fits table newly created
        if (extMap.find(cTableName.str()) == extMap.end())
        {
            for (multimap<string, CCfits::ExtHDU*>::const_iterator it=extMap.begin(); it!= extMap.end(); it++)
                fMess->Debug(it->first);
            return fFile->addTable(cTableName.str(), 0, allNames, allDataTypes, allUnits);
        }
        CCfits::Table* cTable;
        cTable = dynamic_cast<CCfits::Table*>(extMap.find(cTableName.str())->second);

        if (!cTable)
            return NULL;//something wrong happened while getting the table pointer

        //now check that the table columns are the same as the service columns
        cTable->makeThisCurrent();
        const map<string, Column*> cMap = cTable->column();
        for (unsigned int j=0;j<allNames.size();j++)
            if (cMap.find(allNames[j]) == cMap.end())
                continue;

        return cTable;
    }
    fMess->Error("One hundred trials for new table format failed. aborting");
    return NULL;
}
// --------------------------------------------------------------------------
//
//! This opens the FITS file (after the columns have been passed)
//! @param fileName the filename with complete or relative path of the file to open
//! @param tableName the name of the table that will receive the logged data.
//! @param file a pointer to an existing FITS file. If NULL, file will be opened and managed internally
//! @param fitsCounter a pointer to the integer keeping track of the opened FITS files
//! @param out a pointer to the MessageImp that should be used to log errors
//! @param runNumber the runNumber for which this file is opened. 0 means nightly file.
//
bool Fits::Open(const string& fileName, const string& tableName, FITS* file, uint32_t* fitsCounter, MessageImp* out, int runNumber)
{               
//      if (fMess)
//              delete fMess;
//      fMess = new MessageImp(out);
    fRunNumber = runNumber;
        fMess = out;
        fFileName = fileName;
        if (file == NULL)
        {
                try
                {
                        fFile = new FITS(fileName, RWmode::Write);
                }
                catch (CCfits::FitsException e)
                {                       
                        ostringstream str;
                        str << "Opening FITS file " << fileName << ": " << e.message();
                        fMess->Error(str);
                        fFile = NULL;
                        return false;
                }       
                fOwner = true;
                fNumOpenFitsFiles = fitsCounter;
                (*fNumOpenFitsFiles)++;
        }
        else
        {
                fFile = file;
                fOwner = false;
        }
        //concatenate the standard and data columns
        //do it the inneficient way first: its easier and faster to code.
        vector<string> allNames;
        vector<string> allDataTypes;
        vector<string> allUnits;
        fTotalNumBytes = 0;
        for (unsigned int i=0;i<fStandardColDesc.size();i++)
        {
                allNames.push_back(fStandardColDesc[i].name);
                allDataTypes.push_back(fStandardFormats[i]);
                allUnits.push_back(fStandardColDesc[i].unit);
                fTotalNumBytes += fStandardNumBytes[i];
        }
        //for (int i=static_cast<int>(fDataColDesc.size())-1;i>=0;i--)
        for (unsigned int i=0; i<fDataColDesc.size(); i++)
        {
                if (fDataColDesc[i].name != "")
                        allNames.push_back(fDataColDesc[i].name);
                else 
                {
                        ostringstream stt;
                        stt << "Data" << i;
                        allNames.push_back(stt.str());
                }
                allDataTypes.push_back(fDataFormats[i]);
                allUnits.push_back(fDataColDesc[i].unit);       
        }
        fTotalNumBytes += fDataNumBytes;
        
        bool updating = false;
        try
        {
                //first, let's check if the table already exist in the file
                vector<string> tryToLoadName;
                tryToLoadName.push_back(tableName);
                fFile->read(tryToLoadName);

//          const multimap< string, CCfits::ExtHDU * >& extMap = fFile->extension();
//          if (extMap.find(tableName) == extMap.end())
//          {
//              for (multimap<string, CCfits::ExtHDU*>::const_iterator it=extMap.begin(); it!= extMap.end(); it++)
//                  fMess->Debug(it->first);
//              fTable = fFile->addTable(tableName, 0, allNames, allDataTypes, allUnits);
//          }
//          else
//          {
//              fTable = dynamic_cast<CCfits::Table*>(extMap.find(tableName)->second);
//          }
        fTable = findSuitableTableInFitsFile(tableName, allNames, allDataTypes, allUnits);

            if (!fTable)
            {
                fMess->Error("Table " + tableName + " could not be created nor loaded from "+fileName);
                Close();
                return false;
            }
            fTable->makeThisCurrent();
                fCopyBuffer = new unsigned char[fTotalNumBytes]; 
                fNumRows = fTable->rows();
                if (fNumRows !=0)
                {//If the file already existed, then we must load its data to memory before writing to it.
                        BinTable* bTable = dynamic_cast<BinTable*>(fTable);
                        if (!bTable)
                        {
                                fMess->Error("Table " + tableName + " in "+fileName+" could not be converted to a binary table.");
                                Close();
                                return false;
                        }       
                        //read the table binary data.
                        vector<string> colName;
                        bTable->readData(true, colName);

                        //double check that the data was indeed read from the disk. Go through the fTable instead as colName is empty (yes, it is !)
                        const map<string, Column*> cMap = fTable->column();

            for (map<string, Column*>::const_iterator cMapIt = cMap.begin(); cMapIt != cMap.end(); cMapIt++)
                        {
                                if (!cMapIt->second->isRead())
                                {
                                        fMess->Error("Reading column " + cMapIt->first + " back from "+fileName+" failed.");
                                        Close();
                                        return false;
                                }       
                        }
                        updating = true;
                }

        }
        catch(CCfits::FitsException e)
        {
                ostringstream str;
                str << "Opening or creating table " << tableName << " in " << fileName << ": " << e.message();
                fMess->Error(str);
                fTable = NULL;
                Close();
                return false;
        }
                        
        if (!updating)
                return WriteHeaderKeys();

        return true;
}
// --------------------------------------------------------------------------
//
//!This writes a single header key in the currently open file.
//!@param name the key
//!@param value the key value
//!@param a comment explaining the meaning of the key
template <typename T>
bool Fits::WriteSingleHeaderKey(const string &name, const T &value, const string &comment)
{
        try
        {
                fTable->addKey(name, value, comment);
        }
        catch (CCfits::FitsException e)
        {
                ostringstream str;
                str << "Could not add header keys in file " << fFileName << " reason: " << e.message();
                fMess->Error(str);
                return false;
        }
        return true;
}
// --------------------------------------------------------------------------
//
//! This writes the standard header 
//
bool Fits::WriteHeaderKeys()
{
    if (!fTable)
        return false;

    const Time now;
    if (!WriteSingleHeaderKey("EXTREL",   1.0f, "Release Number")) return false;
    if (!WriteSingleHeaderKey("TELESCOP", "FACT", "Telescope that acquired this data")) return false;
    if (!WriteSingleHeaderKey("ORIGIN",   "ISDC", "Institution that wrote the file")) return false;
    if (!WriteSingleHeaderKey("CREATOR",  "fadctrl", "Program that wrote this file (FACT++ datalogger)")) return false;
    if (!WriteSingleHeaderKey("PACKAGE",   PACKAGE_NAME, "Package name")) return false;
    if (!WriteSingleHeaderKey("VERSION",   PACKAGE_VERSION, "Package description")) return false;
    if (!WriteSingleHeaderKey("COMPILED",  __DATE__" "__TIME__, "Compile time")) return false;
    if (!WriteSingleHeaderKey("REVISION",  REVISION, "SVN revision")) return false;
    if (!WriteSingleHeaderKey("DATE",     now.Iso(), "File creation date")) return false;
    if (!WriteSingleHeaderKey("NIGHT",    now.NightAsInt(), "Night as int")) return false;
    if (!WriteSingleHeaderKey("TIMESYS",  "UTC", "Time systen")) return false;
    if (!WriteSingleHeaderKey("TSTART",   "", "Time of the first receied data")) return false;
    if (!WriteSingleHeaderKey("TSTOP",    "", "Time of the last receied data")) return false;
    return true;
}
// --------------------------------------------------------------------------
//
//! This writes one line of data to the file.
//! @param conv the converter corresponding to the service being logged
//
bool Fits::Write(const Converter &conv)
{
    //first copy the standard variables to the copy buffer
    int shift = 0;
    for (unsigned int i=0;i<fStandardNumBytes.size();i++)
    {
        const char *charSrc = reinterpret_cast<char*>(fStandardPointers[i]);
        reverse_copy(charSrc, charSrc+fStandardNumBytes[i], &fCopyBuffer[shift]);
        shift += fStandardNumBytes[i];
    }

    try
    {
        //now take care of the DIM data. The Converter is here for that purpose
        conv.ToFits(&fCopyBuffer[shift], fDataPointer, fDataNumBytes);
    }
    catch (const runtime_error &e)
    {
        ostringstream str;
        str << fFileName << ": " << e.what();
        fMess->Error(str);
        return false;
    }

    fTable->makeThisCurrent();

    int status(0);
    if (fits_insert_rows(fTable->fitsPointer(), fNumRows, 1, &status))
    {
        ostringstream str;
        char text[30];
        fits_get_errstatus(status, text);
        str << "Inserting row into " << fFileName << ": " << text << " (fits_insert_rows, rc=" << status << ")";
        fMess->Error(str);
        Close();
        return false;
    }

    fNumRows++;

    //the first standard variable is the current MjD
    if (fEndMjD==0)
    {
        const double doubleValue = *reinterpret_cast<double*>(fStandardPointers[0]);
        WriteSingleHeaderKey("TSTART", Time(doubleValue).Iso(),
                             "Time of the first received data");
    }
    fEndMjD = *reinterpret_cast<double*>(fStandardPointers[0]);

    //data copied to buffer, can write to fits
    if (fits_write_tblbytes(fFile->fitsPointer(), fNumRows, 1, fTotalNumBytes, fCopyBuffer, &status))
    {
        char text[30];//max length of cfitsio error strings (from doc)
        fits_get_errstatus(status, text);
        ostringstream str;
        str << "Writing FITS row " << fNumRows << " in " << fFileName << ": " << text << " (file_write_tblbytes, rc=" << status << ")";
        fMess->Error(str);
        Close();
        return false;
    }
    return true;
}

// --------------------------------------------------------------------------
//
//! This closes the currently openned FITS file. 
//! it also updates the header to reflect the time of the last logged row
//      
void Fits::Close() 
{
//WARNING: do NOT delete the table as it gets deleted by the fFile object
//                      if (fTable != NULL)
//                              delete fTable;
        if (fFile != NULL && fOwner)
        {
//          CCfits::FITS* backupFits = fFile;
//          fFile = NULL;
            WriteSingleHeaderKey("TSTOP", Time(fEndMjD).Iso(), "Time of the last receied data");
            delete fFile;

        fMess->Info("Closed: "+fFileName);
        if (fNumOpenFitsFiles != NULL)
            (*fNumOpenFitsFiles)--;
        }
        fFile = NULL;
        if (fCopyBuffer != NULL)
                delete [] fCopyBuffer;
        fCopyBuffer = NULL;

//fMess is the MessageImp part of the dataLogger itself. Thus it should NOT be deleted by the Fits files destructor.
//      if (fMess)
//              delete fMess;
        fMess = NULL;
}

// --------------------------------------------------------------------------
//! Returns the size on the disk of the Fits file being written.
int Fits::GetWrittenSize() const
{
        if (!IsOpen())
                return 0;
                
        struct stat st;
        if (stat(fFileName.c_str(), &st))
                return 0;

        return st.st_size;
}