source: trunk/FACT++/src/dataLogger.cc@ 10887

//****************************************************************
/** @class DataLogger
  
  @brief Logs all message and infos between the services
  
  This is the main logging class facility. 
  It derives from StateMachineDim and DimInfoHandler. the first parent is here to enforce 
  a state machine behaviour, while the second one is meant to make the dataLogger receive
  dim services to which it subscribed from.
  The possible states and transitions of the machine are:
  \dot
  digraph datalogger {
          node [shape=record, fontname=Helvetica, fontsize=10];
      e [label="Error" color="red"];
   r [label="Ready"]
   d [label="NightlyOpen"]
   w [label="WaitingRun"]
      l [label="Logging"]
   b [label="BadNightlyconfig" color="red"]
   c [label="BadRunConfig" color="red"]
  
  e -> r
  r -> e
  r -> d
  r -> b
  d -> w
  d -> r
  w -> r
  l -> r
  l -> w
  b -> d
  w -> c
  w -> l
  b -> r
  c -> r
  c -> l
  }
  \enddot
 */
 //****************************************************************
#include "Dim.h"
#include "Event.h"
#include "Time.h"
#include "StateMachineDim.h"
#include "WindowLog.h"
#include "Configuration.h"
#include "ServiceList.h"
#include "Converter.h"
#include "MessageImp.h"
#include "LocalControl.h"
#include "DimDescriptionService.h"

#include "Description.h"

#include "DimServiceInfoList.h"

//for getting stat of opened files
#include <unistd.h>
//for getting disk free space
#include <sys/statvfs.h>
//for getting files sizes
#include <sys/stat.h>

#include <fstream>

#include <boost/bind.hpp>
#if BOOST_VERSION < 104400
#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
#undef BOOST_HAS_RVALUE_REFS
#endif
#endif
#include <boost/thread.hpp>

#ifdef HAVE_FITS
#include "Fits.h"
#endif

//Dim structures
///Distributes the writting statistics
struct DataLoggerStats {
    long sizeWritten;
    long freeSpace;
    long writingRate;
};
///distributes the number of opened subscriptions and fits files
struct NumSubAndFitsType {
    int numSubscriptions;
    int numOpenFits;
};
///distributes which files were opened.
struct OpenFileToDim {
    int code;
    char fileName[FILENAME_MAX];
};

///Run number record. Used to keep track of which run numbers are still active
struct RunNumberType {
    ///the run number log file
    ofstream* logFile;
    ///the run number report file
    ofstream* reportFile;
#ifdef HAVE_FITS
    ///the run number group fits file
    CCfits::FITS* runFitsFile;
#endif
    ///the log filename
    string logName;
    ///the report filename
    string reportName;
    ///the actual run number
    int runNumber;
    ///the time at which the run number was received
    Time time;
    ///internal counter used for smart referencing
    int* numCopies;
    ///list of opened fits used to create the fits grouping when the run ends
    map<string, vector<string> > openedFits;
    ///default constructor
    RunNumberType()
    {
        logFile = new ofstream();
        reportFile = new ofstream();
#ifdef HAVE_FITS
        runFitsFile = NULL;
#endif
        runNumber = -1;
        //give it a meaningless, 0 time to distinguish with actual, valid times
        time = Time(0,0);
        numCopies = new int(1);
    }
    ///default destructor
    ~RunNumberType()
    {
        if (numCopies)
        {
            (*numCopies)--;
            if (*numCopies < 1)
            {
                if (logFile)
                {
                    if (logFile->is_open())
                        logFile->close();
                    delete logFile;
                    logFile = NULL;
                }
                if (reportFile)
                {
                    if (reportFile->is_open())
                        reportFile->close();
                    delete reportFile;
                    reportFile = NULL;
                }
#ifdef HAVE_FITS
                if (runFitsFile)
                {
                    delete runFitsFile;
                    runFitsFile = NULL;
                }
#endif
                delete numCopies;
                numCopies = NULL;
            }
        }
    }
    ///copy operator
    void operator = (const RunNumberType& other)
    {
        logFile = other.logFile;
        reportFile = other.reportFile;
        logName = other.logName;
        reportName = other.reportName;
        runNumber = other.runNumber;
        time = other.time;
        numCopies = other.numCopies;
#ifdef HAVE_FITS
        runFitsFile = other.runFitsFile;
#endif
        (*numCopies)++;
    }
    ///copy constructor
    RunNumberType(const RunNumberType& other)
    {
        logFile = other.logFile;
        reportFile = other.reportFile;
        logName = other.logName;
        reportName = other.reportName;
        runNumber = other.runNumber;
        time = other.time;
        numCopies = other.numCopies;
#ifdef HAVE_FITS
        runFitsFile = other.runFitsFile;
#endif
        (*numCopies)++;
    }

    void addServiceToOpenedFits(const string& fileName, const string& serviceName)
    {//most likely I should add this service name.
     //the only case for which I should not add it is if a service disapeared, hence the file was closed
     //and reopened again. Unlikely to happen, but well it may
        bool found = false;
        for (vector<string>::iterator it=openedFits[fileName].begin(); it!=openedFits[fileName].end(); it++)
            if (*it == serviceName)
            {
                found = true;
                break;
            }
        if (!found)
            openedFits[fileName].push_back(serviceName);
    }

};
///Dim subscription type. Stores all the relevant info to handle a Dim subscription
struct SubscriptionType
{
#ifdef HAVE_FITS
    ///Nightly FITS output file
    Fits    nightlyFile;
    ///run-specific FITS output file
    Fits    runFile;
#endif
    ///the actual dimInfo pointer
    DimStampedInfo* dimInfo;
    ///the converter for outputting the data according to the format
    Converter* fConv;
    ///internal counter used for smart referencing
    int* numCopies;
    ///the current run number used by this subscription
    int runNumber;
    ///copy operator
    void operator = (const SubscriptionType& other)
    {
#ifdef HAVE_FITS
        nightlyFile = other.nightlyFile;
        runFile = other.runFile;
#endif
        dimInfo = other.dimInfo;
        numCopies = other.numCopies;
        fConv = other.fConv;
        runNumber = other.runNumber;
        (*numCopies)++;
    }
    ///copy constructor
    SubscriptionType(const SubscriptionType& other)
    {
#ifdef HAVE_FITS
        nightlyFile = other.nightlyFile;
        runFile = other.runFile;
#endif
        dimInfo = other.dimInfo;
        numCopies = other.numCopies;
        fConv = other.fConv;
        runNumber = other.runNumber;
        (*numCopies)++;
    }
    ///Dim info constructor
    SubscriptionType(DimStampedInfo* info)
    {
        dimInfo = info;
        fConv = NULL;
        runNumber = -1;
        numCopies = new int(1);
    }
    ///default constructor
    SubscriptionType()
    {
        dimInfo = NULL;
        fConv = NULL;
        runNumber = -1;
        numCopies = new int(1);
    }
    ///default destructor
    ~SubscriptionType()
    {
        if (numCopies)
            (*numCopies)--;
        if (numCopies)
        if (*numCopies < 1)
        {
            if (dimInfo)
            delete dimInfo;
#ifdef HAVE_FITS
            if (nightlyFile.IsOpen())
                nightlyFile.Close();
            if (runFile.IsOpen())
                runFile.Close();
#endif
            if (numCopies)
            delete numCopies;
            delete fConv;
            fConv = NULL;
            dimInfo = NULL;
            numCopies = NULL;
        }
    }
};

class DataLogger : public StateMachineDim, DimInfoHandler
{
public:
    /// The list of existing states specific to the DataLogger
    enum
    {
        kSM_NightlyOpen = 20, ///< Nightly file openned and writing
        kSM_WaitingRun = 30, ///< waiting for the run number to open the run file
        kSM_Logging = 40, ///< both files openned and writing
        kSM_BadNightlyConfig = 0x101, ///< the folder specified for Nightly logging does not exist or has bad permissions
        kSM_BadRunConfig = 0x102, ///<  the folder specified for the run logging does not exist or has wrong permissions or no run number
    } localstates_t;
    
    DataLogger(ostream &out);
    ~DataLogger(); 

    bool SetConfiguration(Configuration& conf);

private:
    /************************************************
     * MEMBER VARIABLES
     ************************************************/
    /// ofstream for the NightlyLogfile
    ofstream fNightlyLogFile;
    /// ofstream for the Nightly report file
    ofstream fNightlyReportFile;
    /// base path of the Nightlyfile
    string fNightlyFilePath;
    ///base path of the run file
    string fRunFilePath;
    ///run numbers
    list<RunNumberType> fRunNumber;
    ///previous run number. to check if changed while logging
    int fPreviousRunNumber;
    ///Current Service Quality
    int fQuality;
    ///Modified Julian Date
    double fMjD;
    ///for obtaining the name of the existing services
    ServiceList fServiceList;
    typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
    ///All the services to which we have subscribed to, sorted by server name.
    SubscriptionsListType fServiceSubscriptions;
    ///full name of the nightly log file
    string fFullNightlyLogFileName;
    ///full name of the nightly report file
    string fFullNightlyReportFileName;

public:
    /***************************************************
     * STATIC COMMAND NAMES
     ***************************************************/
    ///Define all the static names
    static const char* fConfigDay;
    static const char* fConfigRun;
    static const char* fConfigRunNumber;
    static const char* fConfigLog;
    static const char* fTransStart;
    static const char* fTransStop;
    static const char* fTransStartRun;
    static const char* fTransStopRun;
    static const char* fTransReset;
    static const char* fTransWait;
    static const char* fRunNumberInfo; ///< This is the name of the dimInfo received to specify the run number. It must be updated once the final name will be defined
    static const char* fPrintCommand;
    static const char* fDebugOnOff;
    static const char* fStatsPeriod;
    static const char* fStartStopOpenedFiles;
    static const char* fStartStopNumSubsAndFits;
private:
    /***************************************************
     * DIM INFO HANDLER
     ***************************************************/
    //overloading of DIM's infoHandler function
    void infoHandler(); 

    /***************************************************
     * TRANSITION FUNCTIONS
     ***************************************************/
    ///Reporting method for the services info received
    void ReportPlease(DimInfo* I, SubscriptionType& sub);  
    ///Configuration of the nightly file path
    int ConfigureNightlyFileName(const Event& evt); 
    ///Configuration fo the file name
    int ConfigureRunFileName(const Event& evt); 
    ///DEPREC - configuration of the run number
    int ConfigureRunNumber(const Event& evt); 
    ///logging method for the messages
//    int LogMessagePlease(const Event& evt);
    ///print the current state of the dataLogger
    int PrintStatePlease(const Event& evt);
    ///checks whether or not the current info being treated is a run number
    void CheckForRunNumber(DimInfo* I); 
    /// start transition
    int StartPlease(); 
    ///from waiting to logging transition
    int StartRunPlease(); 
    /// from logging to waiting transition
    int StopRunPlease(); 
    ///stop and reset transition
    int GoToReadyPlease(); 
    ///from NightlyOpen to waiting transition
    int NightlyToWaitRunPlease(); 
#ifdef HAVE_FITS
    ///Open fits files
    void OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber);
    ///Write data to FITS files
    void WriteToFITS(SubscriptionType& sub);
    ///Allocate the buffers required for fits
    void AllocateFITSBuffers(SubscriptionType& sub);
#endif//has_fits

    /***************************************
     * DIM SERVICES PROVIDED BY THE DATA LOGGER
     ***************************************/
    ///monitoring notification loop
    void ServicesMonitoring();
    inline void NotifyOpenedFile(string name, int type, DimDescribedService* service);
    ///services notification thread
    boost::thread fMonitoringThread;
    ///end of the monitoring
    bool fContinueMonitoring;
    ///stores the size of each file that is or was open
    map<string, long> fFileSizesMap;
    ///total size of the opened files BEFORE they were opened by the logger
    long fBaseSizeNightly;
    long fPreviousSize;
    long fBaseSizeRun;
    ///Service for opened files
    DimDescribedService* fOpenedNightlyFiles;
    DimDescribedService* fOpenedRunFiles;
    DimDescribedService* fNumSubAndFits;
    NumSubAndFitsType fNumSubAndFitsData;
    ///Small function for calculating the total size written so far
    void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting);

    /***************************************************
     * DATA LOGGER's CONFIGURATION STUFF
     ***************************************************/
    ///black/white listing
    set<string> fBlackList;
    set<string> fWhiteList;
    ///list of services to be grouped
    set<string> fGrouping;
    ///configuration flags
    bool fHasBlackList;
    bool fHasWhiteList;
    bool fDebugIsOn;
    float fStatsPeriodDuration;
    bool fOpenedFilesIsOn;
    bool fNumSubAndFitsIsOn;
    //functions for controlling the services behavior
    int SetDebugOnOff(const Event& evt);
    int SetStatsPeriod(const Event& evt);
    int SetOpenedFilesOnOff(const Event& evt);
    int SetNumSubsAndFitsOnOff(const Event& evt);

    ///boolean to prevent DIM update while desctructing the dataLogger
    bool fDestructing;    
    /***************************************************
     * UTILITIES
     ***************************************************/
    ///vectors to keep track of opened Fits files, for grouping purposes.
    map<string, vector<string> > fOpenedNightlyFits;
    ///creates a group fits file based on a list of files to be grouped
    void CreateFitsGrouping(map<string, vector<string> >& filesToGroup, int runNumber);
    ///Open the relevant text files related to a particular run
    int OpenRunFile(RunNumberType& run);
    ///add a new run number
    void AddNewRunNumber(int newRun, Time time);
    ///removes the oldest run number, and close the relevant files.
    void RemoveOldestRunNumber();
    ///checks with fServiceList whether or not the services got updated
    bool CheckForServicesUpdate();
    ///retrieves the size of a file
    off_t GetFileSize(string&);
    ///Form the base (without extension) name of the nightly text files (logs and reports)
    void FormBaseNightlyTextFileName(string& name);
    ///Form the name of the nightly text files (logs and reports)
    void FormNightlyTextFileName(string& name, bool isReport);
    ///Form the base (without extension) name of the run text files (logs and reports)
    void FormBaseRunTextFileName(string& name, const int runNumber);
    ///Form the name of the run text files (logs and reports)
    void FormRunTextFileName(string& name, bool isReport, const int runNumber);
    ///Form the name of the nightly Fits files
    void FormNightlyFitsFileName(string& fileName, string& fileNameWithPath, const string& serviceName);
    ///Form the name of the run Fits files
    void FormRunFitsFileName(string& fileName, string& fileNameWithPath, const string& serviceName, const int runNumber);
    ///Form the name of the grouped fits file (i.e. the file where several services are written)
    void FormGroupedRunFitsFileName(string& fileName, string& fileNameWithPath, const int runNumber);
    ///Form the name of the grouping fits file (i.e. the file that is created after a run ends)
    void FormFitsGroupingFileName(string& fileNameWithPath, const int runNumber);
    ///Check whether service is in black and/or white list
    bool ShouldSubscribe(const string& server, const string& service);
    ///Subscribe to a given server and service
    DimStampedInfo* SubscribeToPlease(const string& server, const string& service);
    ///Open a text file and checks for ofstream status
    void OpenTextFilePlease(ofstream& stream, const string& name);
    ///Check if a dir is . and returns the actual string corresponding to .
    string CheckIfDirIsDot(const string& dir);
    ///Remembers the size of newly opened files. for statistic purposes
    bool RememberFileOrigSizePlease(string& fileName, bool nightly);
    ///Checks if the input osftream is in error state, and if so close it.
    void CheckForOfstreamError(ofstream& out);
}; //DataLogger

// --------------------------------------------------------------------------
//
//! Checks if the given ofstream is in error state and if so, close it
//! @param out the ofstream that should be checked
//
void DataLogger::CheckForOfstreamError(ofstream& out)
{
    if (!out.good())
    {
        Error("An error occured while writing to a text file. Closing it");
        if (out.is_open())
            out.close();
    }
}
// --------------------------------------------------------------------------
//
//! Checks the size on disk of a given size, and remembers it in the relevant member variable
//! @param fileName the file for which the size on disk should be retrieved
//! @param nightly whether this is a run or nightly file, so that its size is added to the correct member variable
//
bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly)
{
    //get the size of the file we're about to open
    if (fFileSizesMap.find(fileName) == fFileSizesMap.end())
    {
        if (nightly)
            fBaseSizeNightly += GetFileSize(fileName);
        else
            fBaseSizeRun += GetFileSize(fileName);
        fFileSizesMap[fileName] = 0;
        return true;
    }
    return false;
}
// --------------------------------------------------------------------------
//
//! Checks if a given directory is simply '.' and retrieves the coresponding full path
//! @param dir the input directory that should be checked against
//! @return the full path corresponding to the input directory
//
string DataLogger::CheckIfDirIsDot(const string& dir)
{
    if (dir == ".")
    {
        char currentPath[FILENAME_MAX];
        if (!getcwd(currentPath, sizeof(currentPath)))
        {
            if (errno != 0)
            {
                ostringstream str;
                str << "Unable to retrieve the current path" << ". Reason: " << strerror(errno) << " [" << errno << "]";
                Error(str);
            }
        }
        return string(currentPath);
    }
    else
    {
        return string(dir);
    }
}
// --------------------------------------------------------------------------
//
//! Open a text file and checks for error code
//! @param stream the ofstream for which the file should be opened
//! @name the file name
//
void DataLogger::OpenTextFilePlease(ofstream& stream, const string& name)
{
    stream.open(name.c_str(), ios_base::out | ios_base::app);
    if (errno != 0)
    {
        ostringstream str;
        str << "Unable to open file: " << name << ". Reason: " << strerror(errno) << " [" << errno << "]";
        Error(str);
    }
}
// --------------------------------------------------------------------------
//
//! Create a new dim subscription to a given server and service
//! @param server the server name
//! @param service the service name
//
DimStampedInfo* DataLogger::SubscribeToPlease(const string& server, const string& service)
{
    if (fDebugIsOn)
    {
        ostringstream str;
        str << "Subscribing to service " << server << "/" << service;
        Debug(str);
    }
    return new DimStampedInfo((server + "/" + service).c_str(), const_cast<char*>(""), this);
}
// --------------------------------------------------------------------------
//
//! Check whether a service should be subscribed to, based on the black/white list entries
//! @param server the server name associated with the service being checked
//! @param service the service name associated with the service being checked
//
bool DataLogger::ShouldSubscribe(const string& server, const string& service)
{
    if (service == "SERVICE_LIST")
        return false;
    if (fHasWhiteList && (fWhiteList.find(server + "/") == fWhiteList.end()) &&
                         (fWhiteList.find(server + "/" + service) == fWhiteList.end()) &&
                         (fWhiteList.find("/" + service) == fWhiteList.end()))
        return false;
    if (fHasBlackList && ((fBlackList.find(server + "/") != fBlackList.end()) ||
                          (fBlackList.find(server + "/" + service) != fBlackList.end()) ||
                          (fBlackList.find("/" + service) != fBlackList.end())))
        return false;

    return true;
}
// --------------------------------------------------------------------------
//
//!Form the name of the grouped fits file (i.e. the file where several services are written)
//! @param fileName only the file name itself (no path)
//! @param fileNameWithPath fileName, augmented with the prefix path
//! @param runNumber the current run number for which the file should be opened.
//
void DataLogger::FormGroupedRunFitsFileName(string& fileName, string& fileNameWithPath, const int runNumber)
{
    ostringstream sRun;
    sRun << runNumber;
    fileName = sRun.str() + "_group.fits";
    fileNameWithPath = fRunFilePath + '/' + fileName;
}
// --------------------------------------------------------------------------
//
//! Form the name of the grouping fits file (i.e. the file that is created after a run ends)
//! This file is meant to link the various fits files related to a particular run from a single file
//! @param fileNameWithPath the full filename
//! @param runNumber the current run number for which the file should be opened. -1 means nightly grouping
//
void DataLogger::FormFitsGroupingFileName(string& fileNameWithPath, const int runNumber)
{
    ostringstream groupName;
    if (runNumber != -1)
    {
        groupName << fRunFilePath << '/' << runNumber << ".fits";
    }
    else
    {
        Time time;
        ostringstream sTime;
        sTime << time.Y() << "_" << time.M() << "_" << time.D();
        groupName << fNightlyFilePath << '/' << sTime.str() << ".fits";
    }
    fileNameWithPath = groupName.str();

}
// --------------------------------------------------------------------------
//
//! Form the name of the nightly Fits files
//! @param fileName the file name only
//! @param fileNameWithPath the fileName augmented by the prefix path
//! @param serviceName the service for which this file should be opened
//
void DataLogger::FormNightlyFitsFileName(string& fileName, string& fileNameWithPath, const string& serviceName)
{
    Time time;
    ostringstream sTime;
    sTime << time.Y() << "_" << time.M() << "_" << time.D();
    fileName = sTime.str() + '_' + serviceName + ".fits";
    fileNameWithPath = fNightlyFilePath + '/' + fileName;
}
// --------------------------------------------------------------------------
//
//! Form the name of the run Fits files
//! @param fileName the file name only
//! @param fileNameWithPath the fileName augmented by the prefix path
//! @param serviceName the service for which this file should be opened
//! @param runNumber the current run number for which this file should be opened
//
void DataLogger::FormRunFitsFileName(string& fileName, string& fileNameWithPath, const string& serviceName, const int runNumber)
{
    ostringstream sRun;
    sRun << runNumber;
    fileName = sRun.str() + '_' + serviceName + ".fits";
    fileNameWithPath = fRunFilePath + '/' + fileName;
}
// --------------------------------------------------------------------------
//
//! Form the base (without extension) name of the run text files (logs and reports)
//! @param name the full file name
//! @param runNumber the current run number for which this file should be opened
//
void DataLogger::FormBaseRunTextFileName(string& name, const int runNumber)
{
    ostringstream sRun;
    sRun << runNumber;

    name = CheckIfDirIsDot(fRunFilePath) + '/' + sRun.str();
}
// --------------------------------------------------------------------------
//
//! Form the name of the run text files (logs and reports)
//! @param name the full file name
//! @param isReport whether a log or report name should be formed
//! @param runNumber the current run number for which this file should be opened
//
void DataLogger::FormRunTextFileName(string& name, bool isReport, const int runNumber)
{
    FormBaseRunTextFileName(name, runNumber);

    if (isReport)
        name += ".rep";
    else
        name += ".log";
}
// --------------------------------------------------------------------------
//
//! Form the base (without extension) name of the nightly text files (logs and reports)
//! @param name the full file name
//
void DataLogger::FormBaseNightlyTextFileName(string& name)
{
    Time time;
    ostringstream sTime;
    sTime << time.Y() << "_" << time.M() << "_" << time.D();

    name = CheckIfDirIsDot(fNightlyFilePath) + '/' + sTime.str();
}
// --------------------------------------------------------------------------
//
//! Form the name of the nightly text files (logs and reports)
//! @param name the full file name
//! @param isReport whether a log or report name should be formed
//
void DataLogger::FormNightlyTextFileName(string& name, bool isReport)
{
    FormBaseNightlyTextFileName(name);

    if (isReport)
        name += ".rep";
    else
        name += ".log";
}
// --------------------------------------------------------------------------
//
//!retrieves the size on disk of a file
//! @param fileName the full file name for which the size on disk should be retrieved
//! @return the size of the file on disk, in bytes. 0 if the file does not exist or if an error occured
//
off_t DataLogger::GetFileSize(string& fileName)
{
    struct stat st;
    if (!stat(fileName.c_str(), &st))
        return st.st_size;

    if (errno != 0)
    {
        ostringstream str;
        str << "Unable to stat " << fileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
        Error(str);
    }

    return 0;
}
// --------------------------------------------------------------------------
//
//! Removes the oldest run number and closes the fits files that should be closed
//! Also creates the fits grouping file
//
void DataLogger::RemoveOldestRunNumber()
{
    if (fDebugIsOn)
    {
        ostringstream str;
        str << "Removing run number " << fRunNumber.front().runNumber;
        Debug(str);
    }
    CreateFitsGrouping(fRunNumber.front().openedFits, fRunNumber.front().runNumber);
    //crawl through the subscriptions to see if there are still corresponding fits files opened.
    SubscriptionsListType::iterator x;
    map<string, SubscriptionType>::iterator y;
    for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
        for (y=x->second.begin(); y != x->second.end(); y++)
            if (y->second.runFile.fRunNumber == fRunNumber.front().runNumber && y->second.runFile.IsOpen())
            {
                if (fDebugIsOn)
                {
                    ostringstream str;
                    str << "Closing Fits run file " << y->second.runFile.fFileName;
                    Debug(str);
                }
                y->second.runFile.Close();
            }
    //if a grouping file is on, decrease the number of opened fits manually
    if (fRunNumber.front().runFitsFile)
        (fNumSubAndFitsData.numOpenFits)--;
    //remove the entry
    fRunNumber.pop_front();
}

// --------------------------------------------------------------------------
//
//! Calculate the total number of written bytes since the logger was started
//! @param statVar the data structure that should be updated
//! @param shouldWarn whether or not error messages should be outputted
//! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information
//
void DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting)
{
#ifdef HAVE_FITS
    if (isPrinting)
    {
        ostringstream str;
        str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:";
        Message(str);
    }
    SubscriptionsListType::iterator x;
    map<string, SubscriptionType>::iterator y;
///TODO the grouping file is dealt with several times. This should not be a problem but well, better to fix it I guess.
    for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
    {
        for (y=x->second.begin(); y != x->second.end(); y++)
        {
            if (y->second.runFile.IsOpen())
            {
                    fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();
                    if (isPrinting)
                        Message("-> "+y->second.runFile.fFileName);
            }
            if (y->second.nightlyFile.IsOpen())
            {
                fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();
                if (isPrinting)
                    Message("-> "+y->second.nightlyFile.fFileName);
            }
        }
    }
#else
    if (isPrinting)
        Message("FITS output disabled at compilation");
#endif
    //gather log and report files sizes on disk
    if (fNightlyLogFile.is_open())
        fFileSizesMap[fFullNightlyLogFileName] = GetFileSize(fFullNightlyLogFileName);
    if (fNightlyReportFile.is_open())
        fFileSizesMap[fFullNightlyReportFileName] = GetFileSize(fFullNightlyReportFileName);
    for (list<RunNumberType>::iterator it = fRunNumber.begin(); it != fRunNumber.end(); it++)
    {
        if (it->reportFile->is_open())
            fFileSizesMap[it->reportName] = GetFileSize(it->reportName);
        if (it->logFile->is_open())
            fFileSizesMap[it->logName] = GetFileSize(it->logName);
    }
    struct statvfs vfs;
    if (!statvfs(fNightlyFilePath.c_str(), &vfs))
    {
        statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
        shouldWarn = false;
    }
    else
    {
        ostringstream str;
        str << "Unable to retrieve stats for " << fNightlyFilePath << ". Reason: " << strerror(errno) << " [" << errno << "]";
        if (!shouldWarn)
            Error(str);
        shouldWarn = true;
        statVar.freeSpace = -1;
    }
    //sum up all the file sizes. past and present
    statVar.sizeWritten = 0;
    for (map<string, long>::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end();  it++)
        statVar.sizeWritten += it->second;
    statVar.sizeWritten -= fBaseSizeNightly;
    statVar.sizeWritten -= fBaseSizeRun;
}
//static members initialization
//since I do not check the transition/config names any longer, indeed maybe these could be hard-coded... but who knows what will happen in the future ?
const char* DataLogger::fConfigDay = "CONFIG_DAY";
const char* DataLogger::fConfigRun = "CONFIG_RUN";
const char* DataLogger::fConfigRunNumber = "CONFIG_RUN_NUMBER";
const char* DataLogger::fConfigLog = "LOG";
const char* DataLogger::fTransStart = "START";
const char* DataLogger::fTransStop = "STOP";
const char* DataLogger::fTransStartRun = "START_RUN";
const char* DataLogger::fTransStopRun = "STOP_RUN";
const char* DataLogger::fTransReset = "RESET";
const char* DataLogger::fTransWait = "WAIT_RUN_NUMBER";
const char* DataLogger::fRunNumberInfo = "RUN_NUMBER";
const char* DataLogger::fPrintCommand = "PRINT";
const char* DataLogger::fDebugOnOff = "DEBUG";
const char* DataLogger::fStatsPeriod = "STATS_PERIOD";
const char* DataLogger::fStartStopOpenedFiles = "OPENED_FILES_SRVC";
const char* DataLogger::fStartStopNumSubsAndFits = "NUM_SUBS_SRVC";

// --------------------------------------------------------------------------
//
//! Monitor the number of opened files and total size written, and distributes this data through a Dim service
//
//
void DataLogger::ServicesMonitoring()
{
        DataLoggerStats statVar;
        statVar.sizeWritten = 0;
        statVar.freeSpace = 0;
        statVar.writingRate = 0;

        struct statvfs vfs;
        if (!statvfs(fNightlyFilePath.c_str(), &vfs))
            statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
        else
            statVar.freeSpace = -1;

        DimDescribedService srvc ("DATA_LOGGER/STATS", "X:3", statVar, "Add description here");
        fPreviousSize = 0;
        bool statWarning = false;
        //loop-wait for broadcast
        while (fContinueMonitoring)
        {
            //check if some run number entries can be deleted
            while (fRunNumber.size() > 1 && (Time() - fRunNumber.front().time) > boost::posix_time::time_duration(0,0,10,0))
            {
                RemoveOldestRunNumber();
            }
            if (fStatsPeriodDuration == 0.0f)
            {
                sleep(0.1f);
                continue;
            }
            else
                sleep(fStatsPeriodDuration);
            //update the fits files sizes
            calculateTotalSizeWritten(statVar, statWarning, false);
            if (fStatsPeriodDuration == 0.0f)
                continue;
            statVar.writingRate = (statVar.sizeWritten - fPreviousSize)/fStatsPeriodDuration;  

            fPreviousSize = statVar.sizeWritten;
            if (statVar.writingRate != 0) //if data has been written
            {
                srvc.updateService();
                if(fDebugIsOn)
                {
                    ostringstream str;
                    str << "Size written: " << statVar.sizeWritten/1000 << " kB; writting rate: ";
                    str << statVar.writingRate/1000 << " kB/s; free space: ";
                    str << statVar.freeSpace/(1000*100) << " MB";
                    Debug(str);
                }
            }
        }
}

// --------------------------------------------------------------------------
//
//! Default constructor. The name of the machine is given DATA_LOGGER
//! and the state is set to kSM_Ready at the end of the function.
//
//!Setup the allows states, configs and transitions for the data logger
//
DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER")
{
    //initialize member data
    fNightlyFilePath = ".";
    fRunFilePath = ".";

    //Give a name to this machine's specific states
    AddStateName(kSM_NightlyOpen,      "NightlyFileOpen",  "The summary files for the night are open.");
    AddStateName(kSM_WaitingRun,       "WaitForRun",       "The summary files for the night are open and we wait for a run to be started.");
    AddStateName(kSM_Logging,          "Logging",          "The summary files for the night and the files for a single run are open.");
    AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "The folder for the nighly summary files is invalid.");
    AddStateName(kSM_BadRunConfig,     "ErrRunFolder",     "The folder for the run files is invalid.");

    /*Add the possible transitions for this machine*/
    AddEvent(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig)
            (boost::bind(&DataLogger::StartPlease, this))
            ("Start the nightly logging. Nightly file location must be specified already");

    AddEvent(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging)
            (boost::bind(&DataLogger::GoToReadyPlease, this))
            ("Stop all data logging, close all files.");

    AddEvent(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig)
            (boost::bind(&DataLogger::StartRunPlease, this))
            ("Start the run logging. Run file location must be specified already.");

    AddEvent(kSM_WaitingRun, fTransStopRun, kSM_Logging)
            (boost::bind(&DataLogger::StopRunPlease, this))
            ("Wait for a run to be started, open run-files as soon as a run number arrives.");

    AddEvent(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_Error)
            (boost::bind(&DataLogger::GoToReadyPlease, this))
            ("Transition to exit error states. Closes the nightly file if already opened.");

    AddEvent(kSM_WaitingRun, fTransWait, kSM_NightlyOpen)
            (boost::bind(&DataLogger::NightlyToWaitRunPlease, this));

    /*Add the possible configurations for this machine*/
    AddEvent(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig)
            (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1))
            ("Configure the folder for the nightly files."
             "|Path[string]:Absolute or relative path name where the nightly files should be stored.");

    AddEvent(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
            (boost::bind(&DataLogger::ConfigureRunFileName, this, _1))
            ("Configure the folder for the run files."
             "|Path[string]:Absolute or relative path name where the run files should be stored.");

    AddEvent(fConfigRunNumber, "I", kSM_Ready, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig, kSM_Logging)
            (boost::bind(&DataLogger::ConfigureRunNumber, this, _1))
            ("configure the run number. cannot be done in logging state");

     //Provide a logging command
     //I get the feeling that I should be going through the EventImp
     //instead of DimCommand directly, mainly because the commandHandler
     //is already done in StateMachineImp.cc
     //Thus I'll simply add a configuration, which I will treat as the logging command
//     AddEvent(fConfigLog, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadRunConfig)
 //            (boost::bind(&DataLogger::LogMessagePlease, this, _1))
 //            ("Log a single message to the log-files."
//              "|Message[string]:Message to be logged.");
        
     //Provide a print command
     ostringstream str;
     str <<     kSM_Ready << " " << kSM_NightlyOpen << " " << kSM_WaitingRun << " " << kSM_Logging << " " << kSM_BadNightlyConfig;
     str << " " << kSM_BadRunConfig;
     AddEvent(fPrintCommand, str.str().c_str(), "")
             (boost::bind(&DataLogger::PrintStatePlease, this, _1))
             ("Print information about the internal status of the data logger.");

     fServiceList.SetHandler(this);
     CheckForServicesUpdate();

     //start the monitoring service
     fContinueMonitoring = true;
     fMonitoringThread = boost::thread(boost::bind(&DataLogger::ServicesMonitoring, this));
     fBaseSizeNightly = 0;
     fBaseSizeRun = 0;
     OpenFileToDim fToDim;
     fToDim.code = 0;
     fToDim.fileName[0] = '\0';

     fOpenedNightlyFiles = new DimDescribedService(GetName() + "/FILENAME_NIGHTLY", "I:1;C", fToDim,
                               "Path and base name which is used to compile the filenames for the nightly files."
                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
                               "|Name[string]:path and base file name");

     fOpenedRunFiles = new DimDescribedService(GetName() + "/FILENAME_RUN", "I:1;C", fToDim,
                               "Path and base name which is used to compile the filenames for the run files."
                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
                               "|Name[string]:path and base file name");

     fNumSubAndFitsData.numSubscriptions = 0;
     fNumSubAndFitsData.numOpenFits = 0;
     fNumSubAndFits = new DimDescribedService(GetName() + "/NUM_SUBS", "I:2", fNumSubAndFitsData,
                               "Shows number of services to which the data logger is currently subscribed and the total number of open files."
                               "|Subscriptions[int]:number of dim services to which the data logger is currently subscribed."
                               "|NumOpenFiles[int]:number of files currently open by the data logger");

     //black/white list
     fHasBlackList = false;
     fHasWhiteList = false;
     fBlackList.clear();
     fWhiteList.clear();
     //services parameters
     fDebugIsOn = false;
     fStatsPeriodDuration = 1.0f;
     fOpenedFilesIsOn = true;
     fNumSubAndFitsIsOn = true;

     //provide services control commands
     AddEvent(fDebugOnOff, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
             (boost::bind(&DataLogger::SetDebugOnOff, this, _1))
             ("Switch debug mode on off. Debug mode prints ifnormation about every service written to a file."
              "|Enable[bool]:Enable of disable debuig mode (yes/no).");

     AddEvent(fStatsPeriod, "F", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
             (boost::bind(&DataLogger::SetStatsPeriod, this, _1))
             ("Interval in which the data-logger statitistics service (STATS) is updated."
              "Interval[s]:Floating point value in seconds.");

     AddEvent(fStartStopOpenedFiles, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
              (boost::bind(&DataLogger::SetOpenedFilesOnOff ,this, _1))
              ("Can be used to switch the service off which distributes information about the open files.");

     AddEvent(fStartStopNumSubsAndFits, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
             (boost::bind(&DataLogger::SetNumSubsAndFitsOnOff, this, _1))
             ("Can be used to switch the service off which distributes information about the number of subscriptions and open files.");

     fDestructing = false;
     if(fDebugIsOn)
     {
         Debug("DataLogger Init Done.");
     }
}
// --------------------------------------------------------------------------
//
//! Checks for changes in the existing services.
//! Any new service will be added to the service list, while the ones which disappeared are removed.
//
//FIXME The service must be udpated so that I get the first notification. This should not be
bool DataLogger::CheckForServicesUpdate()
{ 
    bool serviceUpdated = false;
    //get the current server list
    const vector<string> serverList = fServiceList.GetServerList();
    //first let's remove the servers that may have disapeared
    //can't treat the erase on maps the same way as for vectors. Do it the safe way instead
    vector<string> toBeDeleted;
    for (SubscriptionsListType::iterator cListe = fServiceSubscriptions.begin(); cListe != fServiceSubscriptions.end(); cListe++)
    {
        vector<string>::const_iterator givenServers;
        for (givenServers=serverList.begin(); givenServers!= serverList.end(); givenServers++)
            if (cListe->first == *givenServers)
                break;
        if (givenServers == serverList.end())//server vanished. Remove it
        {    
            toBeDeleted.push_back(cListe->first);
            serviceUpdated = true;
        }
    } 
    for (vector<string>::const_iterator it = toBeDeleted.begin(); it != toBeDeleted.end(); it++)
        fServiceSubscriptions.erase(*it);
    //now crawl through the list of servers, and see if there was some updates
    for (vector<string>::const_iterator i=serverList.begin(); i!=serverList.end();i++)
    {
        //skip the two de-fact excluded services
        //Dim crashes if the publisher subscribes to its own service. This sounds weird, I agree.
        //I agree: hard coded values are bad, but in this case these two entries should always be here otherwise Dim crashes.
        if ((i->find("DIS_DNS") != string::npos) ||
            (i->find("DATA_LOGGER") != string::npos))
            continue;
        //find the current server in our subscription list    
        SubscriptionsListType::iterator cSubs = fServiceSubscriptions.find(*i);
        //get the service list of the current server
        vector<string> cServicesList = fServiceList.GetServiceList(*i);
        if (cSubs != fServiceSubscriptions.end())//if the current server already is in our subscriptions
        {                                         //then check and update our list of subscriptions
            //first, remove the services that may have disapeared.
            map<string, SubscriptionType>::iterator serverSubs;
            vector<string>::const_iterator givenSubs;
            toBeDeleted.clear();
            for (serverSubs=cSubs->second.begin(); serverSubs != cSubs->second.end(); serverSubs++)
            {
                for (givenSubs = cServicesList.begin(); givenSubs != cServicesList.end(); givenSubs++)
                    if (serverSubs->first == *givenSubs)
                        break;
                if (givenSubs == cServicesList.end())
                {
                    toBeDeleted.push_back(serverSubs->first);
                    serviceUpdated = true;
                }    
            }
            for (vector<string>::const_iterator it = toBeDeleted.begin(); it != toBeDeleted.end(); it++)
                cSubs->second.erase(*it);
            //now check for new services
            for (givenSubs = cServicesList.begin(); givenSubs != cServicesList.end(); givenSubs++)
            {
                if (!ShouldSubscribe(*i, *givenSubs))
                    continue;
                if (cSubs->second.find(*givenSubs) == cSubs->second.end())
                {//service not found. Add it
                    cSubs->second[*givenSubs].dimInfo = SubscribeToPlease(*i, *givenSubs);
                    serviceUpdated = true;
                }    
            }
        }
        else //server not found in our list. Create its entry
        {
            fServiceSubscriptions[*i] = map<string, SubscriptionType>();
            map<string, SubscriptionType>& liste = fServiceSubscriptions[*i];
            for (vector<string>::const_iterator j = cServicesList.begin(); j!= cServicesList.end(); j++)
            {
                if (!ShouldSubscribe(*i, *j))
                    continue;
                liste[*j].dimInfo = SubscribeToPlease(*i, *j);
                serviceUpdated = true;
            }
        }    
    }
    return serviceUpdated;
}
// --------------------------------------------------------------------------
//
//! Destructor
//
DataLogger::~DataLogger()
{
    if (fDebugIsOn)
    {
        Debug("DataLogger destruction starts");    
    }
    fDestructing = true;
    //first let's go to the ready state
    GoToReadyPlease(); 
    //release the services subscriptions
    fServiceSubscriptions.clear();
    //exit the monitoring loop
    fContinueMonitoring = false;

    fMonitoringThread.join();
    //clear any remaining run number (should remain only one)
     while (fRunNumber.size() > 0)
     {
         RemoveOldestRunNumber();
     }

    delete fOpenedNightlyFiles;
    delete fOpenedRunFiles;
    delete fNumSubAndFits;

    if (fDebugIsOn)
    {
        Debug("DataLogger desctruction ends");    
    }
}

// --------------------------------------------------------------------------
//
//! Inherited from DimInfo. Handles all the Infos to which we subscribed, and log them
//
void DataLogger::infoHandler()
{
    // Make sure getTimestamp is called _before_ getTimestampMillisecs
    if (fDestructing)
        return;

    DimInfo* I = getInfo();
    SubscriptionsListType::iterator x;
    map<string, SubscriptionType>::iterator y;
    if (I==NULL)
    {
        if (CheckForServicesUpdate())
        {
            //services were updated. Notify
            fNumSubAndFitsData.numSubscriptions = 0;
            for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
                fNumSubAndFitsData.numSubscriptions += x->second.size();
            if (fNumSubAndFitsIsOn)
            {
                if (fDebugIsOn)
                {
                    ostringstream str;
                    str << "Updating number of subscriptions service: Num Subs=" << fNumSubAndFitsData.numSubscriptions << " Num open FITS=" << fNumSubAndFitsData.numOpenFits;
                    Debug(str);
                }
                fNumSubAndFits->updateService();
            }
        }
        return;
    }
    //check if the service pointer corresponds to something that we subscribed to
    //this is a fix for a bug that provides bad Infos when a server starts
    bool found = false;
    for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
    {//find current service is subscriptions
        for (y=x->second.begin(); y!=x->second.end();y++)
            if (y->second.dimInfo == I)
            {
                found = true;    
                break;
            }
        if (found)
            break;
    }
    if (!found)
        return;
    if (I->getSize() <= 0)
        return;

        // Make sure that getTimestampMillisecs is NEVER called before
        // getTimestamp is properly called
        // check that the message has been updated by something, i.e. must be different from its initial value
    if (I->getTimestamp() == 0)
        return;

    CheckForRunNumber(I);

    ReportPlease(I, y->second);

}
// --------------------------------------------------------------------------
//
//! Open the text files associated with the given run number
//! @param run the run number to be dealt with
//
int DataLogger::OpenRunFile(RunNumberType& run)
{
    if (run.logFile->is_open())
    {
        ostringstream str;
        str << "Log file " << run.logName << " was already open when trying to open it in OpenRunFile";
        Error(str);
        return -1;
    }
    FormRunTextFileName(run.logName, false, run.runNumber);
    run.logFile->open(run.logName.c_str(), ios_base::out | ios_base::app);
    if (errno != 0)
    {
        ostringstream str;
        str << "Unable to open run Log " << run.logName << ". Reason: " << strerror(errno) << " [" << errno << "]";
        Error(str);
    }
    //open report file
    FormRunTextFileName(run.reportName, true, run.runNumber);
    if (run.reportFile->is_open())
    {
        ostringstream str;
        str << "Report file " << run.reportName << " was already open when trying to open it in OpenRunFile";
        Error(str);
        return -1;
    }
    run.reportFile->open(run.reportName.c_str(), ios_base::out | ios_base::app);
    if (errno != 0)
    {
        ostringstream str;
        str << "Unable to open run report " << run.reportName << ". Reason: " << strerror(errno) << " [" << errno << "]";
        Error(str);
    }

    if (!run.logFile->is_open() || !run.reportFile->is_open())
    {
        ostringstream str;
        str << "Something went wrong while openning nightly files " << run.logName << " and " << run.reportName;
        Error(str);
        return -1;
    }
    //get the size of the newly opened file.
    RememberFileOrigSizePlease(run.logName, false);
    RememberFileOrigSizePlease(run.reportName, false);

    //TODO this notification scheme might be messed up now.... fix it !
    string baseFileName;
    FormBaseRunTextFileName(baseFileName, run.runNumber);
    NotifyOpenedFile(baseFileName, 3, fOpenedRunFiles);
    run.openedFits.clear();
    return 0;
}
// --------------------------------------------------------------------------
//
//! Add a new active run number
//! @param newRun the new run number
//! @param time the time at which the new run number was issued
//
void DataLogger::AddNewRunNumber(int newRun, Time time)
{
    if (fDebugIsOn)
    {
        ostringstream str;
        str << "Adding new run number " << newRun << " that was issued on " << time;
        Debug(str);
    }
    //Add new run number to run number list
    fRunNumber.push_back(RunNumberType());
    fRunNumber.back().runNumber = newRun;
    fRunNumber.back().time = time;
    if (GetCurrentState() != kSM_Logging)
        return;
    //open the log and report files
    OpenRunFile(fRunNumber.back());
}
// --------------------------------------------------------------------------
//
//! Checks whether or not the current info is a run number.
//! If so, then remember it. A run number is required to open the run-log file
//! @param I
//!        the current DimInfo
//
void DataLogger::CheckForRunNumber(DimInfo* I)
{
    if (strstr(I->getName(), fRunNumberInfo) != NULL)
    {//assumes that the run number is an integer
        AddNewRunNumber(I->getInt(), Time(I->getTimestamp(), I->getTimestampMillisecs()*1000));
        ostringstream str;
        str << "New run number is " << fRunNumber.back().runNumber;
        Message(str);
    }
}

// --------------------------------------------------------------------------
//
//! write infos to log files.
//! @param I
//!     The current DimInfo 
//! @param sub
//!        The dataLogger's subscription corresponding to this DimInfo
//
void DataLogger::ReportPlease(DimInfo* I, SubscriptionType& sub)
{
    //should we log or report this info ? (i.e. is it a message ?)
    bool isItaReport = ((strstr(I->getName(), "Message") == NULL) && (strstr(I->getName(), "MESSAGE") == NULL));
    if (I->getFormat()[0] == 'C')
        isItaReport = false;
    
    if (!fNightlyReportFile.is_open())
        return;

    //create the converter for that service
    if (sub.fConv == NULL && isItaReport)
    {
        //trick the converter in case of 'C'. why do I do this ? well simple: the converter checks that the right number
        //of bytes was written. because I skip 'C' with fits, the bytes will not be allocated, hence the "size copied ckeck"
        //of the converter will fail, hence throwing an exception.
        string fakeFormat(I->getFormat());
        if (fakeFormat[fakeFormat.size()-1] == 'C')
            fakeFormat = fakeFormat.substr(0, fakeFormat.size()-1);
        sub.fConv = new Converter(Out(), I->getFormat());    
        if (!sub.fConv)
        {
            ostringstream str;
            str << "Couldn't properly parse the format... service " << sub.dimInfo->getName() << " ignored.";
            Error(str);
            return;    
        }
    }
    //construct the header
    ostringstream header;
    Time cTime(I->getTimestamp(), I->getTimestampMillisecs()*1000);
    fQuality = I->getQuality();
    fMjD = cTime.Mjd();

    //figure out which run file should be used
    ofstream* targetRunFile = NULL;
    RunNumberType* cRunNumber = NULL;
    if (GetCurrentState() == kSM_Logging)
    {
        list<RunNumberType>::reverse_iterator rit;
        for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
        {
            if (rit->time < cTime) //this is the run number that we want to use
            {
                //Find something better to convert iterator to pointer than the ugly line below....
                cRunNumber = &(*rit);
                sub.runNumber = rit->runNumber;
                targetRunFile = isItaReport ? rit->reportFile : rit->logFile;
                break;
            }
        }
        if (rit == fRunNumber.rend() && fRunNumber.size() != 0)
        {
            ostringstream str;
            str << "Could not find an appropriate run number for info coming at time: " << cTime;
            Error(str);
            Error("Active run numbers: ");
            for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
            {
                str.str("");
                str << rit->runNumber;
                Error(str);
            }
        }
    }

    if (isItaReport)
    {
        //write text header
        header << I->getName() << " " << fQuality << " ";
        header << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
        header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
        header << cTime.ms() << " " << I->getTimestamp() << " ";

        string text;
        try
        {
            text = sub.fConv->GetString(I->getData(), I->getSize());
        }
        catch (const runtime_error &e)
        {
            Out() << kRed << e.what() << endl;
            ostringstream str;
            str << "Could not properly parse the data for service " << sub.dimInfo->getName();
            str << " reason: " << e.what() << ". Entry ignored";
            Error(str);
            return;
        }

        if (text.empty())
        {
            ostringstream str;
            str << "Service " << sub.dimInfo->getName() << " sent an empty string";
            Info(str);
            return;
        }
        //replace bizarre characters by white space
        replace(text.begin(), text.end(), '\n', '\\');
        replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
        
        //write entry to Nightly report
        if (fNightlyReportFile.is_open())
        {
            fNightlyReportFile << header.str() << text << endl;
            CheckForOfstreamError(fNightlyReportFile);
        }
        //write entry to run-report
        if (targetRunFile && targetRunFile->is_open())
        {
            *targetRunFile << header.str() << text << endl;
            CheckForOfstreamError(*targetRunFile);
        }
    }
    else
    {//write entry to both Nightly and run logs
        string n = I->getName();
        ostringstream msg;
        msg << n << ": " << I->getString();

        if (fNightlyLogFile.is_open())
        {
            MessageImp nightlyMess(fNightlyLogFile);
            nightlyMess.Write(cTime, msg.str().c_str(), fQuality);
            CheckForOfstreamError(fNightlyLogFile);
        }
        if (targetRunFile && targetRunFile->is_open())
        {
            MessageImp runMess(*targetRunFile);
            runMess.Write(cTime, msg.str().c_str(), fQuality);
            CheckForOfstreamError(*targetRunFile);
        }
    }

#ifdef HAVE_FITS
    if (isItaReport)
    {
        if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen() || sub.runNumber != sub.runFile.fRunNumber)
            OpenFITSFilesPlease(sub, cRunNumber);
        WriteToFITS(sub);
    }    
#endif

}

// --------------------------------------------------------------------------
//
//! write messages to logs. 
/*
//! @param evt
//!        the current event to log
//! @returns 
//!        the new state. Currently, always the current state
//!
//! @deprecated
//!    I guess that this function should not be any longer
//
//Otherwise re-write it properly with the MessageImp class
int DataLogger::LogMessagePlease(const Event& evt)
{
    if (!fNightlyLogFile.is_open())
        return GetCurrentState();
    Warn("LogMessagePlease has not been checked nor updated since a long while. Undefined behavior to be expected");
    ostringstream header;
    const Time& cTime = evt.GetTime();
    header << evt.GetName() << " " << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
    header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
    header << cTime.ms() << " ";
        
    const Converter conv(Out(), evt.GetFormat());
    if (!conv)
    {
        Error("Couldn't properly parse the format... ignored.");
        return GetCurrentState();
    }

    string text;
    try
    {
        text = conv.GetString(evt.GetData(), evt.GetSize());
    }
    catch (const runtime_error &e)
    {
        Out() << kRed << e.what() << endl;
        Error("Couldn't properly parse the data... ignored.");
        return GetCurrentState();
    }

    if (text.empty())
        return GetCurrentState();

    //replace bizarre characters by white space
    replace(text.begin(), text.end(), '\n', '\\');
    replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
    if (fDebugIsOn)
    {
        ostringstream str;
        str << "Logging: \"" << header << text << "\"";
        Debug(str.str());    
    }
    
    if (fNightlyLogFile.is_open())
    {
        fNightlyLogFile << header;
        if (!fNightlyLogFile.good())
        {
            Error("An error occured while writing to the run log file. Closing it.");
            if (fNightlyLogFile.is_open())
                fNightlyLogFile.close();    
        }
    }
    if (fRunLogFile.is_open())
    {
        fRunLogFile << header;
        if (!fRunLogFile.good())
        {
            Error("An error occured while writing to the run log file. Closing it.");
            if (fRunLogFile.is_open())
                fRunLogFile.close();    
        }
    }
    if (fNightlyLogFile.is_open())
    {
        fNightlyLogFile << text;
        if (!fNightlyLogFile.good())
        {
            Error("An error occured while writing to the run log file. Closing it.");
            if (fNightlyLogFile.is_open())
                fNightlyLogFile.close();    
        }
    }
    if (fRunLogFile.is_open())
    {
        fRunLogFile << text;
        if (!fRunLogFile.good())
        {
            Error("An error occured while writing to the run log file. Closing it.");
            if (fRunLogFile.is_open())
                fRunLogFile.close();    
        }
    }
    return GetCurrentState();
}*/
// --------------------------------------------------------------------------
//
//! print the dataLogger's current state. invoked by the PRINT command
//! @param evt
//!        the current event. Not used by the method
//! @returns 
//!        the new state. Which, in that case, is the current state
//!
int DataLogger::PrintStatePlease(const Event& )
{
    Message("-----------------------------------------");
    Message("------ DATA LOGGER CURRENT STATE --------");
    Message("-----------------------------------------");
    //print the path configuration
    Message("Nightly Path: " + CheckIfDirIsDot(fNightlyFilePath));
    Message("Run Path: " + CheckIfDirIsDot(fRunFilePath));

    //print active run numbers
    ostringstream str;
    str << "Active Run Numbers: ";
    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
        str << "\n" << it->runNumber;
    Message(str);
    Message("----------- OPENED FILES ----------------");
    //print all the open files. 
    if (fNightlyLogFile.is_open())
        Message("Nightly Log..........OPEN");
    else
        Message("Nightly log........CLOSED");
    if (fNightlyReportFile.is_open())
        Message("Nightly Report.......OPEN");
    else
        Message("Nightly Report.....CLOSED");
    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
    {
        if (it->logFile->is_open())
            Message("Run Log " + it->logName + " is OPEN");
        else
            Message("Run Log " + it->logName + " is CLOSED");
        if (it->reportFile->is_open())
            Message("Run Report " + it->reportName + " is OPEN");
        else
            Message("Run Report " + it->reportName + " CLOSED");
    }
    bool statWarning = false;
    DataLoggerStats statVar;
    calculateTotalSizeWritten(statVar, statWarning, false);
    Message("---------------- STATS ------------------");
    str.str("");
    str << "Total Size written: " << statVar.sizeWritten << " bytes.";
        Message(str);
    str.str("");
    str << "Disk free space:    " << statVar.freeSpace   << " bytes.";
        Message(str);
    str.str("");
    str << "Statistics are updated every " << fStatsPeriodDuration << " seconds";
    if (fStatsPeriodDuration != 0)
        Message(str);
    else
        Message("Statistics updates are currently disabled");
    Message("----------- DIM SUBSCRIPTIONS -----------");

    str.str("");
    str << "There are " << fNumSubAndFitsData.numSubscriptions << " active DIM subscriptions:";
    Message(str);

    for (map<const string, map<string, SubscriptionType> >::const_iterator it=fServiceSubscriptions.begin(); it!= fServiceSubscriptions.end();it++)
    {
        Message("Server "+it->first);
        for (map<string, SubscriptionType>::const_iterator it2=it->second.begin(); it2!=it->second.end(); it2++)
            Message(" -> "+it2->first);
    }
    if (fHasBlackList)
    {
        Message("------------- BLOCK LIST ----------------");
        for (set<string>::iterator it=fBlackList.begin(); it != fBlackList.end(); it++)
            Message(*it);
    }
    if (fHasWhiteList)
    {
            Message("----------- ALLOW LIST ------------------");
            for (set<string>::iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++)
                Message(*it);
    }
    if (fGrouping.size() != 0)
    {
            Message("--------- GROUPING LIST -----------------");
            Message("The following servers and/or services will be grouping under a single run fits file:");
            for (set<string>::iterator it=fGrouping.begin(); it != fGrouping.end(); it++)
                Message(*it);
    }
    Message("-----------------------------------------");
    Message("------ END OF DATA LOGGER STATE ---------");
    Message("-----------------------------------------");

    return GetCurrentState();
}

// --------------------------------------------------------------------------
//
//! turn debug mode on and off
//! @param evt
//!        the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1
//! @returns 
//!        the new state. Which, in that case, is the current state
//!
int DataLogger::SetDebugOnOff(const Event& evt)
{
    bool backupDebug = fDebugIsOn;
    fDebugIsOn = evt.GetBool();
    if (fDebugIsOn == backupDebug)
        Warn("Warning: debug mode was already in the requested state");
    else
    {
        ostringstream str;
        str << "Debug mode is now " << fDebugIsOn;
        Message(str);
    }
    return GetCurrentState();
}
// --------------------------------------------------------------------------
//
//! set the statistics update period duration. 0 disables the statistics
//! @param evt
//!        the current event. contains the new duration.
//! @returns 
//!        the new state. Which, in that case, is the current state
//!
int DataLogger::SetStatsPeriod(const Event& evt)
{
    float backupDuration = fStatsPeriodDuration;
    fStatsPeriodDuration = evt.GetFloat();
    if (fStatsPeriodDuration < 0)
    {
        Error("Statistics period duration should be greater than zero. Discarding provided value.");
        fStatsPeriodDuration = backupDuration;
        return GetCurrentState();    
    }
    if (!finite(fStatsPeriodDuration))// != fStatsPeriodDuration)
    {
        Error("Provided duration does not appear to be a valid float. discarding it.");
        fStatsPeriodDuration = backupDuration;
        return GetCurrentState();    
    }
    if (backupDuration == fStatsPeriodDuration)
        Warn("Warning: statistics period was not modified: supplied value already in use");
    else
    {
        if (fStatsPeriodDuration == 0.0f)
            Message("Statistics are now OFF");
        else
        {
            ostringstream str;
            str << "Statistics period is now " << fStatsPeriodDuration << " seconds";
            Message(str);
        }    
    }
    return GetCurrentState();
}
// --------------------------------------------------------------------------
//
//! set the opened files service on or off. 
//! @param evt
//!        the current event. contains the instruction string. similar to setdebugonoff
//! @returns 
//!        the new state. Which, in that case, is the current state
//!
int DataLogger::SetOpenedFilesOnOff(const Event& evt)
{
    bool backupOpened = fOpenedFilesIsOn;
    fOpenedFilesIsOn = evt.GetBool();
    if (fOpenedFilesIsOn == backupOpened)
        Warn("Warning: opened files service mode was already in the requested state");
    else
    {
        ostringstream str;
        str << "Opened files service mode is now " << fOpenedFilesIsOn;
        Message(str);
    }
    return GetCurrentState();
    
}
// --------------------------------------------------------------------------
//
//! set the number of subscriptions and opened fits on and off
//! @param evt
//!        the current event. contains the instruction string. similar to setdebugonoff
//! @returns 
//!        the new state. Which, in that case, is the current state
//!
int DataLogger::SetNumSubsAndFitsOnOff(const Event& evt)
{
    bool backupSubs = fNumSubAndFitsIsOn;
    fNumSubAndFitsIsOn = evt.GetBool();
    if (fNumSubAndFitsIsOn == backupSubs)
        Warn("Warning: Number of subscriptions service mode was already in the requested state");
    else
    {
        ostringstream str;
        str << "Number of subscriptions service mode is now " << fNumSubAndFitsIsOn;
        Message(str);
    }
    return GetCurrentState();
}
// --------------------------------------------------------------------------
//
//!    Sets the path to use for the Nightly log file.
//! @param evt
//!     the event transporting the path
//! @returns
//!        currently only the current state.
//
int DataLogger::ConfigureNightlyFileName(const Event& evt)
{
    if (evt.GetText() != NULL)
    {
        fNightlyFilePath = string(evt.GetText());
        Message("New Nightly folder specified: " + fNightlyFilePath);
    }
    else
        Error("Empty Nightly folder given. Please specify a valid path.");

    return GetCurrentState();
}
// --------------------------------------------------------------------------
//
//! Sets the path to use for the run log file.
//! @param evt
//!        the event transporting the path
//! @returns
//!     currently only the current state
int DataLogger::ConfigureRunFileName(const Event& evt)
{
    if (evt.GetText() != NULL)
    {
        fRunFilePath = string(evt.GetText());
        Message("New Run folder specified: " + fRunFilePath);
    }
    else
        Error("Empty Nightly folder given. Please specify a valid path");

    return GetCurrentState();
}
// --------------------------------------------------------------------------
//
//! Sets the run number.
//! @param evt
//!        the event transporting the run number
//! @returns
//!     currently only the current state
int DataLogger::ConfigureRunNumber(const Event& evt)
{
    AddNewRunNumber(evt.GetInt(), evt.GetTime());
//    fRunNumber = evt.GetInt();
    ostringstream str;
    str << "The new run number is: " << fRunNumber.back().runNumber;
    Message(str);
    return GetCurrentState();
}
// --------------------------------------------------------------------------
//
//! Notifies the DIM service that a particular file was opened
//! @ param name the base name of the opened file, i.e. without path nor extension. 
//!     WARNING: use string instead of string& because I pass values that do not convert to string&.
//!        this is not a problem though because file are not opened so often.
//! @ param type the type of the opened file. 0 = none open, 1 = log, 2 = text, 4 = fits
inline void DataLogger::NotifyOpenedFile(string name, int type, DimDescribedService* service)
{
    if (fOpenedFilesIsOn)
    {
        if (fDebugIsOn)
        {
            ostringstream str;
            str << "Updating files service " << service->getName() << "with code: " << type << " and file: " << name;
            Debug(str);
            str.str("");
            str << "Num subs: " << fNumSubAndFitsData.numSubscriptions << " Num open FITS: " << fNumSubAndFitsData.numOpenFits;
            Debug(str);
        }
        if (name.size()+1 > FILENAME_MAX)
        {
            Error("Provided file name \"" + name + "\" is longer than allowed file name length");
        }
        OpenFileToDim fToDim;
        fToDim.code = type;
        memcpy(fToDim.fileName, name.c_str(), name.size()+1);
        service->setData(reinterpret_cast<void*>(&fToDim), name.size()+1+sizeof(int));
        service->setQuality(0);
        service->updateService();
    }
}
// --------------------------------------------------------------------------
//
//! Implements the Start transition.
//! Concatenates the given path for the Nightly file and the filename itself (based on the day), 
//! and tries to open it.
//! @returns 
//!        kSM_NightlyOpen if success, kSM_BadNightlyConfig if failure
int DataLogger::StartPlease()
{
    if (fDebugIsOn)
    {
        Debug("Starting...");    
    }

    FormNightlyTextFileName(fFullNightlyLogFileName, false);
    OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName);

    FormNightlyTextFileName(fFullNightlyReportFileName, true);
    OpenTextFilePlease(fNightlyReportFile, fFullNightlyReportFileName);

    if (!fNightlyLogFile.is_open() || !fNightlyReportFile.is_open())
    {    
        ostringstream str;
        str << "Something went wrong while openning nightly files " << fFullNightlyLogFileName << " and " << fFullNightlyReportFileName;
        Error(str);
        return kSM_BadNightlyConfig;
    }
    //get the size of the newly opened file.
    fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName);
    fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName);
    fFileSizesMap.clear();
    fBaseSizeRun = 0;
    fPreviousSize = 0;

    //notify that a new file has been opened.
    string baseFileName;
    FormBaseNightlyTextFileName(baseFileName);
    NotifyOpenedFile(baseFileName, 3, fOpenedNightlyFiles);

    fOpenedNightlyFits.clear();
    
    return kSM_NightlyOpen;     
}

#ifdef HAVE_FITS
// --------------------------------------------------------------------------
//
//! open if required a the FITS files corresponding to a given subscription
//! @param sub
//!     the current DimInfo subscription being examined
void DataLogger::OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber)
{
    string serviceName(sub.dimInfo->getName());
    //if run number has changed, reopen a new fits file with the correct run number.
     if (sub.runFile.IsOpen() && sub.runFile.fRunNumber != sub.runNumber)
     {
         if (fDebugIsOn)
             Debug("Run number changed. Closing " + sub.runFile.fFileName);
         sub.runFile.Close();
     }
      //we must check if we should group this service subscription to a single fits file before we replace the / by _
    bool hasGrouping = false;
    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
    {//will we find this service in the grouping list ?
        for (set<string>::iterator it=fGrouping.begin(); it!=fGrouping.end(); it++)
        {
            if (serviceName.find(*it) != string::npos)
            {
                hasGrouping = true;
                break;
            }
        }
    }
    hasGrouping = true;
    for (unsigned int i=0;i<serviceName.size(); i++)
    {
        if (serviceName[i] == '/')
        {
            serviceName[i] = '_';
            break;    
        }    
    }
    //we open the NightlyFile anyway, otherwise this function shouldn't have been called.
    if (!sub.nightlyFile.IsOpen())
    {
        string fileNameOnly, partialName;
        FormNightlyFitsFileName(fileNameOnly, partialName, serviceName);
        AllocateFITSBuffers(sub);
        //get the size of the file we're about to open
        if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update)
            fOpenedNightlyFits[fileNameOnly].push_back(serviceName);

        sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, -1);//Out());

        //notify the opening
        string baseFileName;
        FormBaseNightlyTextFileName(baseFileName);
        NotifyOpenedFile(baseFileName, 7, fOpenedNightlyFiles);
        if (fNumSubAndFitsIsOn)
            fNumSubAndFits->updateService();
        if (fDebugIsOn)
        {
            ostringstream str;
            str << "Opened Nightly FITS: " << partialName << " and table: FACT-" << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
            Debug(str);
        }
    }
    //do the actual file open
    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging) && sub.runNumber != -1)
    {//buffer for the run file have already been allocated when doing the Nightly file
        string fileNameOnly;
        string partialName;

        if (hasGrouping)
        {
            FormGroupedRunFitsFileName(fileNameOnly, partialName, sub.runNumber);
        }
        else
        {
            FormRunFitsFileName(fileNameOnly, partialName, serviceName, sub.runNumber);
        }
        //get the size of the file we're about to open
        if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update)
            cRunNumber->openedFits[fileNameOnly].push_back(serviceName);
        else
            if (hasGrouping)
            {
             cRunNumber->addServiceToOpenedFits(fileNameOnly, serviceName);
            }

        if (hasGrouping && cRunNumber->runFitsFile == NULL)
            try
            {
                cRunNumber->runFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write);
                (fNumSubAndFitsData.numOpenFits)++;
            }    
            catch (CCfits::FitsException e)
            {
                ostringstream str;
                str << "Could not open FITS Run file " << partialName << " reason: " << e.message();
                Error(str);
                cRunNumber->runFitsFile = NULL;
            }

        string baseFileName;
        FormBaseRunTextFileName(baseFileName, sub.runNumber);
        NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles);// + '_' + serviceName, 4);
        if (hasGrouping)
            sub.runFile.Open(partialName, serviceName, cRunNumber->runFitsFile, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber);//Out());
        else
            sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber);//Out());

       if (fNumSubAndFitsIsOn)
           fNumSubAndFits->updateService();
           if (fDebugIsOn)
        {
            ostringstream str;
            str << "Opened Run FITS: " << partialName << " and table: FACT-" << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
            Debug(str);
        }
    }
}    
// --------------------------------------------------------------------------
//
//! Allocates the required memory for a given pair of fits files (nightly and run)
//! @param sub the subscription of interest.
//
void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
{
    int size = sub.dimInfo->getSize();
     
    //Init the time columns of the file
    Description dateDesc(string("Time"), string("Modified Julian Date"), string("MjD"));
    sub.nightlyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
    sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));

    Description QoSDesc("Qos", "Quality of service", "None");
    sub.nightlyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
    sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));

    const Converter::FormatList flist = sub.fConv->GetList();
    // Compilation failed
    if (!sub.fConv->valid())
    {
        Error("Compilation of format string failed.");
        return;
    }

    //we've got a nice structure describing the format of this service's messages.
    //Let's create the appropriate FITS columns
    vector<string> dataFormatsLocal;
    for (unsigned int i=0;i<flist.size()-1;i++)
    {
         ostringstream dataQualifier;

         dataQualifier << flist[i].second.first;
         switch (flist[i].first.first->name()[0])
         {
             case 'c':
             case 'C':
                 dataQualifier.str("S");
             break;
             case 's':
                 dataQualifier << "I";
             break;
             case 'i':
             case 'I':
                 dataQualifier << "J";
             break;
             case 'l':
             case 'L':
                 dataQualifier << "J";
             break;
             case 'f':
             case 'F':
                 dataQualifier << "E";
             break;
             case 'd':
             case 'D':
                 dataQualifier << "D";
             break;
             case 'x':
             case 'X':
                 dataQualifier << "K";
             break;
             case 'S':
                 //for strings, the number of elements I get is wrong. Correct it
                 dataQualifier.str(""); //clear
                 dataQualifier << size-1 <<  "A";
                 size = size-1;
             break;
             
             default:
                 Fatal("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 1198.");
         };
         //we skip the variable length strings for now (in fits only)
         if (dataQualifier.str() != "S")
             dataFormatsLocal.push_back(dataQualifier.str());
     }
     sub.nightlyFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size);
     sub.runFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size);
}
// --------------------------------------------------------------------------
//
//! write a dimInfo data to its corresponding FITS files
//
void DataLogger::WriteToFITS(SubscriptionType& sub)
{
        //nightly File status (open or not) already checked
        if (sub.nightlyFile.IsOpen())
        {
            sub.nightlyFile.Write(sub.fConv);
            if (fDebugIsOn)
            {
                Debug("Writing to nightly FITS " + sub.nightlyFile.fFileName);    
            }
        }
        if (sub.runFile.IsOpen())
        {
            sub.runFile.Write(sub.fConv);
            if (fDebugIsOn)
            {
                Debug("Writing to Run FITS " + sub.runFile.fFileName);    
            }
        }
}
#endif //if has_fits
// --------------------------------------------------------------------------
//
//! Implements the StartRun transition.
//! Concatenates the given path for the run file and the filename itself (based on the run number),
//! and tries to open it.
//! @returns
//!        kSM_Logging if success, kSM_BadRunConfig if failure.
int DataLogger::StartRunPlease()
{
    if (fDebugIsOn)
    {
        Debug("Starting Run Logging...");    
    }
    //open all the relevant run-files. i.e. all the files associated with run numbers.
    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
        OpenRunFile(*it);

    return kSM_Logging;
}

#ifdef HAVE_FITS
// --------------------------------------------------------------------------
//
//! Create a fits group file with all the run-fits that were written (either daily or run)
//! @param filesToGroup a map of filenames mapping to table names to be grouped (i.e. a
//!        single file can contain several tables to group
//! @param runNumber the run number that should be used for grouping. -1 means nightly group
//
void DataLogger::CreateFitsGrouping(map<string, vector<string> > & filesToGroup, int runNumber)
{
    if (fDebugIsOn)
    {
        ostringstream str;
        str << "Creating fits group for ";
        if (runNumber != -1)
            str << "run files";
        else
            str << "nightly files";
        Debug(str);
    }
    //create the FITS group corresponding to the ending run.
    CCfits::FITS* groupFile;
    unsigned int numFilesToGroup = 0;
    for (map<string, vector<string> >::iterator it=filesToGroup.begin(); it != filesToGroup.end(); it++)
    {
        numFilesToGroup += it->second.size();
    }
    if (fDebugIsOn)
    {
        ostringstream str;
        str << "There are " << numFilesToGroup << " tables to group";
        Debug(str);
    }
    if (numFilesToGroup <= 1)
    {
        filesToGroup.clear();
        return;
    }
    string groupName;
    FormFitsGroupingFileName(groupName, runNumber);
    CCfits::Table* groupTable;
    int maxCharLength = 50;//FILENAME_MAX;
    try
    {
        groupFile = new CCfits::FITS(groupName, CCfits::RWmode::Write);
        //setup the column names
        ostringstream pathTypeName;
        pathTypeName << maxCharLength << "A";
        vector<string> names;
        vector<string> dataTypes;
        names.push_back("MEMBER_XTENSION");
        dataTypes.push_back("8A");
        names.push_back("MEMBER_URI_TYPE");
        dataTypes.push_back("3A");
        names.push_back("MEMBER_LOCATION");
        dataTypes.push_back(pathTypeName.str());
        names.push_back("MEMBER_NAME");
        dataTypes.push_back(pathTypeName.str());

        groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
//TODO handle the case when the logger was stopped and restarted during the same day, i.e. the grouping file must be updated
     }
     catch (CCfits::FitsException e)
     {
         ostringstream str;
         str << "Could not open or create FITS table GROUPING in  file " << groupName << " reason: " << e.message();
         Error(str);
         return;
     }

    //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about.
    //use cfitsio routines instead
    groupTable->makeThisCurrent();
    //create appropriate buffer.
    unsigned char* fitsBuffer = new unsigned char[8 + 3 + 2*maxCharLength + 1]; //+1 for trailling character
    memset(fitsBuffer, 0, 8 + 3 + 2*maxCharLength + 1);
    char* startOfExtension = reinterpret_cast<char*>(fitsBuffer);
    char* startOfURI = reinterpret_cast<char*>(&fitsBuffer[8]);
    char* startOfLocation = reinterpret_cast<char*>(&fitsBuffer[8 + 3]);
    char* startOfName = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]);

    sprintf(startOfExtension, "%s", "BINTABLE");
    sprintf(startOfURI, "%s", "URL");
    int i=1;
    for (map<string, vector<string> >::iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++)
        for (vector<string>::iterator jt=it->second.begin(); jt != it->second.end(); jt++, i++)
        {
            strcpy(startOfLocation, it->first.c_str());
            strcpy(startOfName, jt->c_str());
            if (fDebugIsOn)
            {
                ostringstream str;
                str << "Grouping " << it->first << " " << *jt;
                Debug(str);
            }
            int status = 0;
            fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status);
            if (status)
            {
                ostringstream str;
                str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status;
                Error(str);
            }
        }

    filesToGroup.clear();
    delete groupFile;
}
#endif //HAVE_FITS

// --------------------------------------------------------------------------
//
//! Implements the StopRun transition.
//! Attempts to close the run file.
//! @returns
//!        kSM_WaitingRun if success, kSM_FatalError otherwise
int DataLogger::StopRunPlease()
{

    if (fDebugIsOn)
    {
        Debug("Stopping Run Logging...");    
    }
    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
    {
        if (!it->logFile->is_open() || !it->reportFile->is_open())
            return kSM_FatalError;
        it->logFile->close();
        it->reportFile->close();
    }

#ifdef HAVE_FITS
    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
        {
            if (j->second.runFile.IsOpen())
                j->second.runFile.Close();
        }
#endif
    NotifyOpenedFile("", 0, fOpenedRunFiles);
    if (fNumSubAndFitsIsOn)
        fNumSubAndFits->updateService();

    while (fRunNumber.size() > 0)
    {
        RemoveOldestRunNumber();
    }

    return kSM_WaitingRun;
}
// --------------------------------------------------------------------------
//
//! Implements the Stop and Reset transitions.
//! Attempts to close any openned file.
//! @returns
//!     kSM_Ready
int DataLogger::GoToReadyPlease()
{
   if (fDebugIsOn)
    {
        Debug("Going to the Ready state...");
    }    
   if (GetCurrentState() == kSM_Logging)
       StopRunPlease();

    if (fNightlyLogFile.is_open())
        fNightlyLogFile.close();
    if (fNightlyReportFile.is_open())
        fNightlyReportFile.close();
        
#ifdef HAVE_FITS
    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
        {
            if (j->second.nightlyFile.IsOpen())
                j->second.nightlyFile.Close();;
        }
#endif
    if (GetCurrentState() == kSM_Logging || 
        GetCurrentState() == kSM_WaitingRun || 
        GetCurrentState() == kSM_NightlyOpen)
    { 
        NotifyOpenedFile("", 0, fOpenedNightlyFiles);
        if (fNumSubAndFitsIsOn)
            fNumSubAndFits->updateService();
    }
#ifdef HAVE_FITS
    CreateFitsGrouping(fOpenedNightlyFits, -1);
#endif
    return kSM_Ready;
}
// --------------------------------------------------------------------------
//
//! Implements the transition towards kSM_WaitingRun
//! Does nothing really.
//!    @returns
//!        kSM_WaitingRun
int DataLogger::NightlyToWaitRunPlease()
{
    if (fDebugIsOn)
    {
        Debug("Going to Wait Run Number state...");    
    }
    return kSM_WaitingRun;    
}
// --------------------------------------------------------------------------
//
//! Setup Logger's configuration from a Configuration object
//! @param conf the configuration object that should be used
//!
bool DataLogger::SetConfiguration(Configuration& conf)
{
    fDebugIsOn = conf.Get<bool>("debug");

    //Set the block or allow list
    fBlackList.clear();
    fWhiteList.clear();
    if (conf.Has("block"))
    {
        vector<string> vec = conf.Get<vector<string>>("block");
        if (vec.size() != 0)
        {
            fHasBlackList = true;
            if (fDebugIsOn)
                Debug("Setting BLOCK list:");
        }
        for (vector<string>::iterator it = vec.begin(); it != vec.end(); it++)
        {
            fBlackList.insert(*it);
            if (fDebugIsOn)
                Debug("                   " + *it);
        }
//Adding entries that should ALWAYS be ignored, because of Dim: otherwise the DataLogger would crash
        fBlackList.insert("DATA_LOGGER/");
        fBlackList.insert("/SERVICE_LIST");
        fBlackList.insert("DIS_DNS/");
    }
    if (conf.Has("allow"))
    {
        vector<string> vec = conf.Get<vector<string>>("allow");
        if (vec.size() != 0)
        {
            fHasWhiteList = true;
            if (fDebugIsOn)
                Debug("Setting ALLOW list:");
        }
        for (vector<string>::iterator it=vec.begin(); it != vec.end(); it++)
        {
            fWhiteList.insert(*it);
            if (fDebugIsOn)
                Debug("                   " + *it);
        }
    }
    //Set the grouping
    if (conf.Has("group"))
    {
        vector<string> vec = conf.Get<vector<string>>("group");
        if (vec.size() != 0)
            if (fDebugIsOn)
                Debug("Setting GROUPING list:");
        for (vector<string>::iterator it=vec.begin(); it != vec.end(); it++)
        {
            fGrouping.insert(*it);
            if (fDebugIsOn)
                Debug("                   " + *it);
        }
    }
    return true;
}

// --------------------------------------------------------------------------
int RunDim(Configuration &conf)
{
    WindowLog wout;

    //log.SetWindow(stdscr);
    if (conf.Has("log"))
        if (!wout.OpenLogFile(conf.Get<string>("log")))
            wout << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;

    // Start io_service.Run to use the StateMachineImp::Run() loop
    // Start io_service.run to only use the commandHandler command detaching
    DataLogger logger(wout);
    if (!logger.SetConfiguration(conf))
        return -1;

    logger.Run(true);

    return 0;
}
// --------------------------------------------------------------------------
void RunThread(DataLogger* logger)
{
    // This is necessary so that the StateMachine Thread can signal the 
    // Readline thread to exit
    logger->Run(true);
    Readline::Stop();    
}
// --------------------------------------------------------------------------
template<class T>
int RunShell(Configuration &conf)
{
    static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);

    WindowLog &win  = shell.GetStreamIn();
    WindowLog &wout = shell.GetStreamOut();

    if (conf.Has("log"))
        if (!wout.OpenLogFile(conf.Get<string>("log")))
            win << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;

    DataLogger logger(wout);
    if (!logger.SetConfiguration(conf))
        return -1;
    
    shell.SetReceiver(logger);

    boost::thread t(boost::bind(RunThread, &logger));
    
    shell.Run(); // Run the shell
    
    logger.Stop();
    
    //Wait until the StateMachine has finished its thread
    //before returning and destroyinng the dim objects which might 
    //still be in use.
    t.join();

    return 0;
}

/*
 Extract usage clause(s) [if any] for SYNOPSIS.
 Translators: "Usage" and "or" here are patterns (regular expressions) which
 are used to match the usage synopsis in program output.  An example from cp
 (GNU coreutils) which contains both strings:
  Usage: cp [OPTION]... [-T] SOURCE DEST
    or:  cp [OPTION]... SOURCE... DIRECTORY
    or:  cp [OPTION]... -t DIRECTORY SOURCE...
 */
void PrintUsage()
{
    cout << "\n"
        "The data logger connects to all available Dim services and "
        "writes them to ascii and fits files.\n"
        "\n"
        "The default is that the program is started without user interaction. "
        "All actions are supposed to arrive as DimCommands. Using the -c "
        "option, a local shell can be initialized. With h or help a short "
        "help message about the usage can be brought to the screen.\n"
        "\n"
        "Usage: dataLogger [-c type] [OPTIONS]\n"
        "  or:  dataLogger [OPTIONS]\n";
    cout << endl;

}
// --------------------------------------------------------------------------
void PrintHelp()
{
    /* Additional help text which is printed after the configuration
     options goes here */
    cout <<
        "\n"
        "The block option has priority over the allow, "
        "i.e. if both are present, only the block list is kept. "
        "If only a server name or service without its server prefix is given "
        "then all the services of that server, or all the services that "
        "correspond to the given suffix are ignored or considered.\n"
        "\n"
        "For example, block=DIS_DNS will skip all the services offered by "
        "the DIS_DNS server, while block=SERVICE_LIST will skip all the "
        "SERVICE_LIST services offered by any server.\n"
        "\n"
        "The commands offered by the dataLoger are the following: \n";
    cout << setw(20) << DataLogger::fConfigDay << " : specify the path where to put the nightly files\n";
    cout << setw(20) << DataLogger::fConfigRun << " : specify the path where to put the run files\n";
    cout << setw(20) << DataLogger::fConfigRunNumber << " : specify the run number\n";
    cout << setw(20) << DataLogger::fConfigLog << " : log a particular message\n";
    cout << setw(20) << DataLogger::fTransStart << " : start the nightly logging\n";
    cout << setw(20) << DataLogger::fTransStop << " : stop the nightly logging\n";
    cout << setw(20) << DataLogger::fTransStartRun << " : start the run logging\n";
    cout << setw(20) << DataLogger::fTransStopRun << " : stop the run logging\n";
    cout << setw(20) << DataLogger::fTransReset << " : stop any logging and/or recover from an error state\n";
    cout << setw(20) << DataLogger::fTransWait << " : go to the wait for run number state\n";
    cout << setw(20) << DataLogger::fPrintCommand << " : print the current state of the logger to the shell\n";
    cout << setw(20) << DataLogger::fDebugOnOff << " : turn on or off the debug mode\n";
    cout << setw(20) << DataLogger::fStatsPeriod << " : set the periodicity of the statistics. 0 disable them\n";
    cout << endl;
}
// --------------------------------------------------------------------------
void SetupConfiguration(Configuration &conf)
{
    const string n = conf.GetName()+".log";

    po::options_description configp("Program options");
    configp.add_options()
        ("dns",       var<string>("localhost"),  "Dim nameserver host name (Overwites DIM_DNS_NODE environment variable)")
        ("log,l",     var<string>(n), "Write log-file")
        ("console,c", var<int>(),     "Use console (0=shell, 1=simple buffered, X=simple unbuffered)")
        ;

    po::options_description configs("DataLogger options");
    configs.add_options()
        ("block,b", vars<string>(), "Black-list of services")
        ("allow,a", vars<string>(), "White-list of services")
        ("debug",   po_bool(),      "Debug mode. Print clear text of received service reports to log-stream")
        ("group,g", vars<string>(), "Grouping of services into a single run-Fits")
        ;

    conf.AddEnv("dns", "DIM_DNS_NODE");

    conf.AddOptions(configp);
    conf.AddOptions(configs);
}
// --------------------------------------------------------------------------
int main(int argc, const char* argv[])
{
    Configuration conf(argv[0]);
    conf.SetPrintUsage(PrintUsage);
    SetupConfiguration(conf);

    po::variables_map vm;
    try
    {
        vm = conf.Parse(argc, argv);
    }
#if BOOST_VERSION > 104000
    catch (po::multiple_occurrences &e)
    {
        cerr << "Program options invalid due to: " << e.what() << " of '" << e.get_option_name() << "'." << endl;
        return -1;
    }
#endif
    catch (exception& e)
    {
        cerr << "Program options invalid due to: " << e.what() << endl;
        return -1;
    }

    if (conf.HasVersion() || conf.HasPrint())
        return -1;

    if (conf.HasHelp())
    {
        PrintHelp();
        return -1;
    }

    Dim::Setup(conf.Get<string>("dns"));

//    try
    {
        // No console access at all
        if (!conf.Has("console"))
            return RunDim(conf);

        // Console access w/ and w/o Dim
        if (conf.Get<int>("console")==0)
            return RunShell<LocalShell>(conf);
        else
            return RunShell<LocalConsole>(conf);
    }
/*    catch (exception& e)
    {
        cerr << "Exception: " << e.what() << endl;
        return -1;
    }*/

    return 0;
}