Changeset 10814


Ignore:
Timestamp:
May 25, 2011, 1:18:26 PM (9 years ago)
Author:
lyard
Message:
Upgrade of the dataLogger to take several run numbers on the fly
Location:
trunk/FACT++/src
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/FACT++/src/Fits.cc

    r10780 r10814  
    8282//! @param fitsCounter a pointer to the integer keeping track of the opened FITS files
    8383//! @param out a pointer to the MessageImp that should be used to log errors
    84 //
    85 void Fits::Open(const string& fileName, const string& tableName, FITS* file, int* fitsCounter, MessageImp* out)//ostream& out)
     84//! @param runNumber the runNumber for which this file is opened. -1 means nightly file.
     85//
     86void Fits::Open(const string& fileName, const string& tableName, FITS* file, int* fitsCounter, MessageImp* out, int runNumber)
    8687{               
    8788//      if (fMess)
    8889//              delete fMess;
    8990//      fMess = new MessageImp(out);
     91    fRunNumber = runNumber;
    9092        fMess = out;
    9193        fFileName = fileName;
  • trunk/FACT++/src/Fits.h

    r10741 r10814  
    5959                ///were to log the errors
    6060                MessageImp* fMess;             
    61         public:
     61public:
     62        ///current run number being logged
     63        int fRunNumber;
    6264               
    6365                Fits() :  fFile(NULL),
     
    7375                                         fFileName(""),
    7476                                         fNumOpenFitsFiles(NULL),
    75                                          fMess(NULL)
     77                                         fMess(NULL),
     78                                         fRunNumber(-1)
    7679                 {}
    7780               
     
    9194
    9295                ///Opens a FITS file
    93         void Open(const string& fileName, const string& tableName, CCfits::FITS* file, int* fitsCounter, MessageImp* out);//ostream& out);
     96        void Open(const string& fileName, const string& tableName, CCfits::FITS* file, int* fitsCounter, MessageImp* out, int runNumber);//ostream& out);
    9497
    9598                ///Write one line of data. Use the given converter.
  • trunk/FACT++/src/dataLogger.cc

    r10780 r10814  
    7878
    7979//Dim structures
     80///Distributes the writting statistics
    8081struct DataLoggerStats {
    8182    long sizeWritten;
     
    8384    long writingRate;
    8485};
    85 
     86///distributes the number of opened subscriptions and fits files
    8687struct NumSubAndFitsType {
    8788    int numSubscriptions;
    8889    int numOpenFits;
    8990};
    90 
     91///distributes which files were opened.
    9192struct OpenFileToDim {
    9293    int code;
    9394    char fileName[FILENAME_MAX];
     95};
     96
     97///Run number record. Used to keep track of which run numbers are still active
     98struct RunNumberType {
     99    ///the run number log file
     100    ofstream* logFile;
     101    ///the run number report file
     102    ofstream* reportFile;
     103#ifdef HAVE_FITS
     104    ///the run number group fits file
     105    CCfits::FITS* runFitsFile;
     106#endif
     107    ///the log filename
     108    string logName;
     109    ///the report filename
     110    string reportName;
     111    ///the actual run number
     112    int runNumber;
     113    ///the time at which the run number was received
     114    Time time;
     115    ///internal counter used for smart referencing
     116    int* numCopies;
     117    ///list of opened fits used to create the fits grouping when the run ends
     118    map<string, vector<string> > openedFits;
     119    ///default constructor
     120    RunNumberType()
     121    {
     122        logFile = new ofstream();
     123        reportFile = new ofstream();
     124#ifdef HAVE_FITS
     125        runFitsFile = NULL;
     126#endif
     127        runNumber = -1;
     128        time = Time(0,0);
     129        numCopies = new int(1);
     130    }
     131    ///default destructor
     132    ~RunNumberType()
     133    {
     134        if (numCopies)
     135        {
     136            (*numCopies)--;
     137            if (*numCopies < 1)
     138            {
     139                if (logFile)
     140                {
     141                    if (logFile->is_open())
     142                        logFile->close();
     143                    delete logFile;
     144                    logFile = NULL;
     145                }
     146                if (reportFile)
     147                {
     148                    if (reportFile->is_open())
     149                        reportFile->close();
     150                    delete reportFile;
     151                    reportFile = NULL;
     152                }
     153#ifdef HAVE_FITS
     154                if (runFitsFile)
     155                {
     156                    delete runFitsFile;
     157                    runFitsFile = NULL;
     158                }
     159#endif
     160                delete numCopies;
     161                numCopies = NULL;
     162            }
     163        }
     164    }
     165    ///copy operator
     166    void operator = (const RunNumberType& other)
     167    {
     168        logFile = other.logFile;
     169        reportFile = other.reportFile;
     170        logName = other.logName;
     171        reportName = other.reportName;
     172        runNumber = other.runNumber;
     173        time = other.time;
     174        numCopies = other.numCopies;
     175#ifdef HAVE_FITS
     176        runFitsFile = other.runFitsFile;
     177#endif
     178        (*numCopies)++;
     179    }
     180    ///copy constructor
     181    RunNumberType(const RunNumberType& other)
     182    {
     183        logFile = other.logFile;
     184        reportFile = other.reportFile;
     185        logName = other.logName;
     186        reportName = other.reportName;
     187        runNumber = other.runNumber;
     188        time = other.time;
     189        numCopies = other.numCopies;
     190#ifdef HAVE_FITS
     191        runFitsFile = other.runFitsFile;
     192#endif
     193        (*numCopies)++;
     194    }
     195
     196};
     197///Dim subscription type. Stores all the relevant info to handle a Dim subscription
     198struct SubscriptionType
     199{
     200#ifdef HAVE_FITS
     201    ///Nightly FITS output file
     202    Fits    nightlyFile;
     203    ///run-specific FITS output file
     204    Fits    runFile;
     205#endif
     206    ///the actual dimInfo pointer
     207    DimStampedInfo* dimInfo;
     208    ///the converter for outputting the data according to the format
     209    Converter* fConv;
     210    ///internal counter used for smart referencing
     211    int* numCopies;
     212    ///the current run number used by this subscription
     213    int runNumber;
     214    ///copy operator
     215    void operator = (const SubscriptionType& other)
     216    {
     217#ifdef HAVE_FITS
     218        nightlyFile = other.nightlyFile;
     219        runFile = other.runFile;
     220#endif
     221        dimInfo = other.dimInfo;
     222        numCopies = other.numCopies;
     223        fConv = other.fConv;
     224        runNumber = other.runNumber;
     225        (*numCopies)++;
     226    }
     227    ///copy constructor
     228    SubscriptionType(const SubscriptionType& other)
     229    {
     230#ifdef HAVE_FITS
     231        nightlyFile = other.nightlyFile;
     232        runFile = other.runFile;
     233#endif
     234        dimInfo = other.dimInfo;
     235        numCopies = other.numCopies;
     236        fConv = other.fConv;
     237        runNumber = other.runNumber;
     238        (*numCopies)++;
     239    }
     240    ///Dim info constructor
     241    SubscriptionType(DimStampedInfo* info)
     242    {
     243        dimInfo = info;
     244        fConv = NULL;
     245        runNumber = -1;
     246        numCopies = new int(1);
     247    }
     248    ///default constructor
     249    SubscriptionType()
     250    {
     251        dimInfo = NULL;
     252        fConv = NULL;
     253        runNumber = -1;
     254        numCopies = new int(1);
     255    }
     256    ///default destructor
     257    ~SubscriptionType()
     258    {
     259        if (numCopies)
     260            (*numCopies)--;
     261        if (numCopies)
     262        if (*numCopies < 1)
     263        {
     264            if (dimInfo)
     265            delete dimInfo;
     266#ifdef HAVE_FITS
     267            if (nightlyFile.IsOpen())
     268                nightlyFile.Close();
     269            if (runFile.IsOpen())
     270                runFile.Close();
     271#endif
     272            if (numCopies)
     273            delete numCopies;
     274            delete fConv;
     275            fConv = NULL;
     276            dimInfo = NULL;
     277            numCopies = NULL;
     278        }
     279    }
    94280};
    95281
     
    109295    DataLogger(ostream &out);
    110296    ~DataLogger();
    111    
     297
     298    bool SetConfiguration(Configuration& conf);
     299
    112300private:
    113     //Define all the data structure specific to the DataLogger here
    114      /// ofstream for the NightlyLogfile
     301    /************************************************
     302     * MEMBER VARIABLES
     303     ************************************************/
     304    /// ofstream for the NightlyLogfile
    115305    ofstream fNightlyLogFile;
    116     /// ofstream for the run-specific Log file
    117     ofstream fRunLogFile;
    118 
    119306    /// ofstream for the Nightly report file
    120307    ofstream fNightlyReportFile;
    121     /// ofstream for the run-specific report file
    122     ofstream fRunReportFile;
    123308    /// base path of the Nightlyfile
    124     string fNightlyFileName;
     309    string fNightlyFilePath;
    125310    ///base path of the run file
    126     string fRunFileName;
    127     ///run number (-1 means no run number specified)
    128     int fRunNumber;
     311    string fRunFilePath;
     312    ///run numbers
     313    list<RunNumberType> fRunNumber;
    129314    ///previous run number. to check if changed while logging
    130315    int fPreviousRunNumber;
     
    133318    ///Modified Julian Date
    134319    double fMjD;
     320    ///for obtaining the name of the existing services
     321    ServiceList fServiceList;
     322    typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
     323    ///All the services to which we have subscribed to, sorted by server name.
     324    SubscriptionsListType fServiceSubscriptions;
     325    ///full name of the nightly log file
     326    string fFullNightlyLogFileName;
     327    ///full name of the nightly report file
     328    string fFullNightlyReportFileName;
     329
    135330public:
     331    /***************************************************
     332     * STATIC COMMAND NAMES
     333     ***************************************************/
    136334    ///Define all the static names
    137335    static const char* fConfigDay;
     
    152350    static const char* fStartStopNumSubsAndFits;
    153351private:
     352    /***************************************************
     353     * DIM INFO HANDLER
     354     ***************************************************/
    154355    //overloading of DIM's infoHandler function
    155356    void infoHandler();
    156    
    157     ///for obtaining the name of the existing services
    158     ServiceList fServiceList;
    159    
    160     ///A std pair to store both the DimInfo pointer and the corresponding outputted fits file
    161     struct SubscriptionType
    162     {
    163 #ifdef HAVE_FITS
    164         ///Nightly FITS output file
    165         Fits    nightlyFile;
    166         ///run-specific FITS output file
    167         Fits    runFile;
    168 #endif
    169         ///the actual dimInfo pointer
    170         DimStampedInfo* dimInfo;
    171         ///the converter for outputting the data according to the format
    172         Converter* fConv;
    173         ///the number of existing handlers to this structure.
    174         ///This is required otherwise I MUST handle the deleting of dimInfo outside from the destructor
    175         int* numCopies;
    176         void operator = (const SubscriptionType& other)
    177         {
    178 #ifdef HAVE_FITS
    179             nightlyFile = other.nightlyFile;
    180             runFile = other.runFile;
    181 #endif
    182             dimInfo = other.dimInfo;   
    183             numCopies = other.numCopies;
    184             fConv = other.fConv;
    185             (*numCopies)++;
    186         }
    187         SubscriptionType(const SubscriptionType& other)
    188         {
    189 #ifdef HAVE_FITS
    190             nightlyFile = other.nightlyFile;
    191             runFile = other.runFile;
    192 #endif
    193             dimInfo = other.dimInfo;
    194             numCopies = other.numCopies;
    195             fConv = other.fConv;
    196             (*numCopies)++;   
    197         }
    198         SubscriptionType(DimStampedInfo* info)
    199         {
    200             dimInfo = info;   
    201             fConv = NULL;
    202             numCopies = new int(1);
    203         }
    204         SubscriptionType()
    205         {
    206             dimInfo = NULL;
    207             fConv = NULL;
    208             numCopies = new int(1);
    209         }
    210         ~SubscriptionType()
    211         {
    212             if (numCopies)
    213             (*numCopies)--;
    214             if (numCopies)
    215             if (*numCopies < 1)
    216             {
    217                 if (dimInfo)
    218                 delete dimInfo;
    219 #ifdef HAVE_FITS
    220                 if (nightlyFile.IsOpen())
    221                     nightlyFile.Close();
    222                 if (runFile.IsOpen())
    223                     runFile.Close();
    224 #endif
    225                 if (numCopies)   
    226                 delete numCopies;
    227                 delete fConv;
    228                 fConv = NULL;
    229                 dimInfo = NULL;
    230                 numCopies = NULL;
    231             }
    232         }
    233     };
    234     typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
    235     ///All the services to which we have subscribed to, sorted by server name.
    236     SubscriptionsListType fServiceSubscriptions;
    237 
     357
     358    /***************************************************
     359     * TRANSITION FUNCTIONS
     360     ***************************************************/
    238361    ///Reporting method for the services info received
    239362    void ReportPlease(DimInfo* I, SubscriptionType& sub); 
    240 
    241363    ///Configuration of the nightly file path
    242364    int ConfigureNightlyFileName(const Event& evt);
     
    246368    int ConfigureRunNumber(const Event& evt);
    247369    ///logging method for the messages
    248     int LogMessagePlease(const Event& evt);
     370//    int LogMessagePlease(const Event& evt);
    249371    ///print the current state of the dataLogger
    250372    int PrintStatePlease(const Event& evt);
     
    263385#ifdef HAVE_FITS
    264386    ///Open fits files
    265     void OpenFITSFilesPlease(SubscriptionType& sub);
     387    void OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber);
    266388    ///Write data to FITS files
    267389    void WriteToFITS(SubscriptionType& sub);
    268390    ///Allocate the buffers required for fits
    269391    void AllocateFITSBuffers(SubscriptionType& sub);
    270     ///FITS file for runs grouping. only one, hence dealt with in the dataLogger itself
    271     CCfits::FITS* fRunFitsFile;
    272392#endif//has_fits
    273 public:   
    274     ///checks with fServiceList whether or not the services got updated
    275     bool CheckForServicesUpdate();
    276 
    277 private:   
     393
     394    /***************************************
     395     * DIM SERVICES PROVIDED BY THE DATA LOGGER
     396     ***************************************/
    278397    ///monitoring notification loop
    279398    void ServicesMonitoring();
     399    inline void NotifyOpenedFile(string name, int type, DimDescribedService* service);
    280400    ///services notification thread
    281401    boost::thread fMonitoringThread;
    282402    ///end of the monitoring
    283403    bool fContinueMonitoring;
    284     ///required for accurate monitoring
     404    ///stores the size of each file that is or was open
    285405    map<string, long> fFileSizesMap;
    286     string fFullNightlyLogFileName;
    287     string fFullNightlyReportFileName;
    288     string fFullRunLogFileName;
    289     string fFullRunReportFileName;
     406    ///total size of the opened files BEFORE they were opened by the logger
    290407    long fBaseSizeNightly;
    291408    long fPreviousSize;
     
    296413    DimDescribedService* fNumSubAndFits;
    297414    NumSubAndFitsType fNumSubAndFitsData;
    298 
    299     inline void NotifyOpenedFile(string name, int type, DimDescribedService* service);
    300 
    301 public:   
    302     bool SetConfiguration(Configuration& conf);
    303 
    304 private:
     415    ///Small function for calculating the total size written so far
     416    void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting);
     417
     418    /***************************************************
     419     * DATA LOGGER's CONFIGURATION STUFF
     420     ***************************************************/
     421    ///black/white listing
    305422    set<string> fBlackList;
    306423    set<string> fWhiteList;
     424    ///list of services to be grouped
    307425    set<string> fGrouping;
     426    ///configuration flags
    308427    bool fHasBlackList;
    309428    bool fHasWhiteList;
     
    317436    int SetOpenedFilesOnOff(const Event& evt);
    318437    int SetNumSubsAndFitsOnOff(const Event& evt);
     438
    319439    ///boolean to prevent DIM update while desctructing the dataLogger
    320440    bool fDestructing;   
    321 
    322     ///Small function for calculating the total size written so far
    323     void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting);
    324 
     441    /***************************************************
     442     * UTILITIES
     443     ***************************************************/
    325444    ///vectors to keep track of opened Fits files, for grouping purposes.
    326     //This cannot be done otherwise, as services may disapear before the files are nicely closed. Hence which files were opened must be remembered.
    327     map<string, vector<string> > fOpenedRunFits;
    328445    map<string, vector<string> > fOpenedNightlyFits;
    329     void CreateFitsGrouping(bool runGroup);
     446    ///creates a group fits file based on a list of files to be grouped
     447    void CreateFitsGrouping(map<string, vector<string> >& filesToGroup, int runNumber);
     448    ///Open the relevant text files related to a particular run
     449    int OpenRunFile(RunNumberType& run);
     450    ///add a new run number
     451    void AddNewRunNumber(int newRun, Time time);
     452    ///removes the oldest run number, and close the relevant files.
     453    void RemoveOldestRunNumber();
     454    ///checks with fServiceList whether or not the services got updated
     455    bool CheckForServicesUpdate();
    330456}; //DataLogger
    331457
     458// --------------------------------------------------------------------------
     459//
     460//! Removes the oldest run number and closes the fits files that should be closed
     461//! Also creates the fits grouping file
     462//
     463void DataLogger::RemoveOldestRunNumber()
     464{
     465    if (fDebugIsOn)
     466    {
     467        stringstream str;
     468        str << "Removing run number " << fRunNumber.front().runNumber;
     469        Debug(str.str());
     470    }
     471    CreateFitsGrouping(fRunNumber.front().openedFits, fRunNumber.front().runNumber);
     472    //crawl through the subscriptions to see if there are still corresponding fits files opened.
     473    SubscriptionsListType::iterator x;
     474    map<string, SubscriptionType>::iterator y;
     475    for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
     476        for (y=x->second.begin(); y != x->second.end(); y++)
     477            if (y->second.runFile.fRunNumber == fRunNumber.front().runNumber && y->second.runFile.IsOpen())
     478            {
     479                if (fDebugIsOn)
     480                {
     481                    stringstream str;
     482                    str << "Closing Fits run file " << y->second.runFile.fFileName;
     483                    Debug(str.str());
     484                }
     485                y->second.runFile.Close();
     486            }
     487    //if a grouping file is on, decrease the number of opened fits manually
     488    if (fRunNumber.front().runFitsFile)
     489        (fNumSubAndFitsData.numOpenFits)--;
     490    //remove the entry
     491    fRunNumber.pop_front();
     492}
     493
     494// --------------------------------------------------------------------------
     495//
     496//! Calculate the total number of written bytes since the logger was started
     497//! @param statVar the data structure that should be updated
     498//! @param shouldWarn whether or not error messages should be outputted
     499//! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information
     500//
    332501void DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting)
    333502{
     
    377546        fFileSizesMap[fFullNightlyReportFileName] = st.st_size;
    378547    }
    379     if (fRunLogFile.is_open())
    380     {
    381         stat(fFullRunLogFileName.c_str(), &st);
    382         fFileSizesMap[fFullRunLogFileName] = st.st_size;
    383     }
    384     if (fRunReportFile.is_open())
    385     {
    386         stat(fFullRunReportFileName.c_str(), &st);
    387         fFileSizesMap[fFullRunReportFileName] = st.st_size;
     548    for (list<RunNumberType>::iterator it = fRunNumber.begin(); it != fRunNumber.end(); it++)
     549    {
     550        if (it->reportFile->is_open())
     551        {
     552            stat(it->reportName.c_str(), &st);
     553            fFileSizesMap[it->reportName] = st.st_size;
     554        }
     555        if (it->logFile->is_open())
     556        {
     557            stat(it->logName.c_str(), &st);
     558            fFileSizesMap[it->logName] = st.st_size;
     559        }
    388560    }
    389561    struct statvfs vfs;
    390     if (!statvfs(fNightlyFileName.c_str(), &vfs))
     562    if (!statvfs(fNightlyFilePath.c_str(), &vfs))
    391563    {
    392564        statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
     
    397569        ostringstream str;
    398570        str.str("");
    399         str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
     571        str << "Unable to retrieve stats for " << fNightlyFilePath << ". Reason: " << strerror(errno) << " [" << errno << "]";
    400572        if (!shouldWarn)
    401573            Error(str);
     
    429601const char* DataLogger::fStartStopNumSubsAndFits = "NUM_SUBS_SRVC";
    430602
     603// --------------------------------------------------------------------------
     604//
     605//! Monitor the number of opened files and total size written, and distributes this data through a Dim service
     606//
     607//
    431608void DataLogger::ServicesMonitoring()
    432609{
     
    437614
    438615        struct statvfs vfs;
    439         if (!statvfs(fNightlyFileName.c_str(), &vfs))
     616        if (!statvfs(fNightlyFilePath.c_str(), &vfs))
    440617            statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
    441618        else
     
    448625        while (fContinueMonitoring)
    449626        {
     627            //check if some run number entries can be deleted
     628            while (fRunNumber.size() > 1 && (Time() - fRunNumber.front().time) > boost::posix_time::time_duration(0,0,10,0))
     629            {
     630                RemoveOldestRunNumber();
     631            }
    450632            if (fStatsPeriodDuration == 0.0f)
    451633            {
     
    477659}
    478660
    479 
    480661// --------------------------------------------------------------------------
    481662//
     
    490671    dis_disable_padding();
    491672    //initialize member data
    492     fNightlyFileName = ".";
    493     fRunFileName = ".";
    494     fRunNumber = -1;
    495     fPreviousRunNumber = fRunNumber;
    496 #ifdef HAVE_FITS
    497     fRunFitsFile = NULL;
    498 #endif
    499 
    500      //Give a name to this machine's specific states
    501      AddStateName(kSM_NightlyOpen,      "NightlyFileOpen",  "The summary files for the night are open.");
    502      AddStateName(kSM_WaitingRun,       "WaitForRun",       "The summary files for the night are open and we wait for a run to be started.");
    503      AddStateName(kSM_Logging,          "Logging",          "The summary files for the night and the files for a single run are open.");
    504      AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "The folder for the nighly summary files is invalid.");
    505      AddStateName(kSM_BadRunConfig,     "ErrRunFolder",     "The folder for the run files is invalid.");
    506 
    507      /*Add the possible transitions for this machine*/
    508      AddEvent(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig)
    509              (boost::bind(&DataLogger::StartPlease, this))
    510              ("Start the nightly logging. Nightly file location must be specified already");
    511 
    512      AddEvent(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging)
    513              (boost::bind(&DataLogger::GoToReadyPlease, this))
    514              ("Stop all data logging, close all files.");
    515 
    516      AddEvent(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig)
    517              (boost::bind(&DataLogger::StartRunPlease, this))
    518              ("Start the run logging. Run file location must be specified already.");
    519 
    520      AddEvent(kSM_WaitingRun, fTransStopRun, kSM_Logging)
    521              (boost::bind(&DataLogger::StopRunPlease, this))
    522              ("Wait for a run to be started, open run-files as soon as a run number arrives.");
    523 
    524      AddEvent(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_Error)
    525              (boost::bind(&DataLogger::GoToReadyPlease, this))
    526              ("Transition to exit error states. Closes the nightly file if already opened.");
    527 
    528      AddEvent(kSM_WaitingRun, fTransWait, kSM_NightlyOpen)
    529              (boost::bind(&DataLogger::NightlyToWaitRunPlease, this));
    530 
    531      /*Add the possible configurations for this machine*/
    532      AddEvent(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig)
    533              (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1))
    534              ("Configure the folder for the nightly files."
    535               "|Path[string]:Absolute or relative path name where the nightly files should be stored.");
    536 
    537      AddEvent(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
    538              (boost::bind(&DataLogger::ConfigureRunFileName, this, _1))
    539              ("Configure the folder for the run files."
    540               "|Path[string]:Absolute or relative path name where the run files should be stored.");
    541 
    542      AddEvent(fConfigRunNumber, "I", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
    543              (boost::bind(&DataLogger::ConfigureRunNumber, this, _1))
    544              ("configure the run number. cannot be done in logging state");
     673    fNightlyFilePath = ".";
     674    fRunFilePath = ".";
     675
     676    //Give a name to this machine's specific states
     677    AddStateName(kSM_NightlyOpen,      "NightlyFileOpen",  "The summary files for the night are open.");
     678    AddStateName(kSM_WaitingRun,       "WaitForRun",       "The summary files for the night are open and we wait for a run to be started.");
     679    AddStateName(kSM_Logging,          "Logging",          "The summary files for the night and the files for a single run are open.");
     680    AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "The folder for the nighly summary files is invalid.");
     681    AddStateName(kSM_BadRunConfig,     "ErrRunFolder",     "The folder for the run files is invalid.");
     682
     683    /*Add the possible transitions for this machine*/
     684    AddEvent(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig)
     685            (boost::bind(&DataLogger::StartPlease, this))
     686            ("Start the nightly logging. Nightly file location must be specified already");
     687
     688    AddEvent(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging)
     689            (boost::bind(&DataLogger::GoToReadyPlease, this))
     690            ("Stop all data logging, close all files.");
     691
     692    AddEvent(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig)
     693            (boost::bind(&DataLogger::StartRunPlease, this))
     694            ("Start the run logging. Run file location must be specified already.");
     695
     696    AddEvent(kSM_WaitingRun, fTransStopRun, kSM_Logging)
     697            (boost::bind(&DataLogger::StopRunPlease, this))
     698            ("Wait for a run to be started, open run-files as soon as a run number arrives.");
     699
     700    AddEvent(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_Error)
     701            (boost::bind(&DataLogger::GoToReadyPlease, this))
     702            ("Transition to exit error states. Closes the nightly file if already opened.");
     703
     704    AddEvent(kSM_WaitingRun, fTransWait, kSM_NightlyOpen)
     705            (boost::bind(&DataLogger::NightlyToWaitRunPlease, this));
     706
     707    /*Add the possible configurations for this machine*/
     708    AddEvent(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig)
     709            (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1))
     710            ("Configure the folder for the nightly files."
     711             "|Path[string]:Absolute or relative path name where the nightly files should be stored.");
     712
     713    AddEvent(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
     714            (boost::bind(&DataLogger::ConfigureRunFileName, this, _1))
     715            ("Configure the folder for the run files."
     716             "|Path[string]:Absolute or relative path name where the run files should be stored.");
     717
     718    AddEvent(fConfigRunNumber, "I", kSM_Ready, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig, kSM_Logging)
     719            (boost::bind(&DataLogger::ConfigureRunNumber, this, _1))
     720            ("configure the run number. cannot be done in logging state");
    545721
    546722     //Provide a logging command
     
    549725     //is already done in StateMachineImp.cc
    550726     //Thus I'll simply add a configuration, which I will treat as the logging command
    551      AddEvent(fConfigLog, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadRunConfig)
    552              (boost::bind(&DataLogger::LogMessagePlease, this, _1))
    553              ("Log a single message to the log-files."
    554               "|Message[string]:Message to be logged.");
     727//     AddEvent(fConfigLog, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadRunConfig)
     728 //            (boost::bind(&DataLogger::LogMessagePlease, this, _1))
     729 //            ("Log a single message to the log-files."
     730//              "|Message[string]:Message to be logged.");
    555731       
    556732     //Provide a print command
     
    768944    if (fNightlyReportFile.is_open())
    769945        fNightlyReportFile.close();
    770     if (fRunLogFile.is_open())
    771         fRunLogFile.close();
    772     if (fRunReportFile.is_open())
    773         fRunReportFile.close();
     946    //check if some run number entries can be deleted
     947     while (fRunNumber.size() > 0)
     948     {
     949         RemoveOldestRunNumber();
     950     }
     951
    774952    delete fOpenedNightlyFiles;
    775953    delete fOpenedRunFiles;
    776954    delete fNumSubAndFits;
    777 #ifdef HAVE_FITS
    778     if (fRunFitsFile != NULL)
    779         delete fRunFitsFile;
    780     fRunFitsFile = NULL;
    781 #endif
     955
    782956    if (fDebugIsOn)
    783957    {
     
    8471021    CheckForRunNumber(I);
    8481022
    849     if (fPreviousRunNumber != fRunNumber)
    850     {//run number has changed. close and reopen run files.
    851         StopRunPlease();
    852         StartRunPlease();
    853         fPreviousRunNumber = fRunNumber;
    854     }
    855 
    8561023    ReportPlease(I, y->second);
    8571024
    8581025}
    859 
     1026// --------------------------------------------------------------------------
     1027//
     1028//! Open the text files associated with the given run number
     1029//! @param run the run number to be dealt with
     1030//
     1031int DataLogger::OpenRunFile(RunNumberType& run)
     1032{
     1033    ostringstream sRun;
     1034    sRun << run.runNumber;
     1035    run.logName = fRunFilePath + '/' + sRun.str() + ".log";
     1036    if (run.logFile->is_open())
     1037    {
     1038        stringstream str;
     1039        str << "Log file " << run.logName << " was already open when trying to open it in OpenRunFile";
     1040        Error(str.str());
     1041        return -1;
     1042    }
     1043
     1044    run.logFile->open(run.logName.c_str(), ios_base::out | ios_base::app);
     1045    if (errno != 0)
     1046    {
     1047        ostringstream str;
     1048        str << "Unable to open run Log " << run.logName << ". Reason: " << strerror(errno) << " [" << errno << "]";
     1049        Error(str);
     1050    }
     1051    //open report file
     1052    run.reportName = fRunFilePath + '/' + sRun.str() + ".rep";
     1053    if (run.reportFile->is_open())
     1054    {
     1055        stringstream str;
     1056        str << "Report file " << run.reportName << " was already open when trying to open it in OpenRunFile";
     1057        Error(str.str());
     1058        return -1;
     1059    }
     1060    run.reportFile->open(run.reportName.c_str(), ios_base::out | ios_base::app);
     1061    if (errno != 0)
     1062    {
     1063        ostringstream str;
     1064        str << "Unable to open run report " << run.reportName << ". Reason: " << strerror(errno) << " [" << errno << "]";
     1065        Error(str);
     1066    }
     1067
     1068    if (!run.logFile->is_open() || !run.reportFile->is_open())
     1069    {
     1070        ostringstream str;
     1071        str << "Something went wrong while openning nightly files " << run.logName << " and " << run.reportName;
     1072        Error(str.str());
     1073        return -1;
     1074    }
     1075    //get the size of the newly opened file.
     1076    struct stat st;
     1077    fBaseSizeRun = 0;
     1078    if (fFileSizesMap.find(run.logName) == fFileSizesMap.end())
     1079    {
     1080        stat(run.logName.c_str(), &st);
     1081        if (errno != 0)
     1082        {
     1083            ostringstream str;
     1084            str << "Unable to stat " << run.logName << ". Reason: " << strerror(errno) << " [" << errno << "]";
     1085            Error(str);
     1086        }
     1087        else
     1088            fBaseSizeRun += st.st_size;
     1089        fFileSizesMap[run.logName] = 0;
     1090    }
     1091    if (fFileSizesMap.find(run.reportName) == fFileSizesMap.end())
     1092    {
     1093        stat(run.reportName.c_str(), &st);
     1094        if (errno != 0)
     1095        {
     1096            ostringstream str;
     1097            str << "Unable to stat " << run.reportName << ". Reason: " << strerror(errno) << " [" << errno << "]";
     1098            Error(str);
     1099        }
     1100        else
     1101            fBaseSizeRun += st.st_size;
     1102        fFileSizesMap[run.reportName] = 0;
     1103    }
     1104    string actualTargetDir;
     1105    if (fRunFilePath == ".")
     1106    {
     1107        char currentPath[FILENAME_MAX];
     1108        if (!getcwd(currentPath, sizeof(currentPath)))
     1109        {
     1110            if (errno != 0)
     1111            {
     1112                ostringstream str;
     1113                str << "Unable to retrieve the current path" << ". Reason: " << strerror(errno) << " [" << errno << "]";
     1114                Error(str);
     1115            }
     1116        }
     1117        actualTargetDir = currentPath;
     1118    }
     1119    else
     1120    {
     1121        actualTargetDir = fRunFilePath;
     1122    }
     1123//TODO this notification scheme might be messed up now.... fix it !
     1124    NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 3, fOpenedRunFiles);
     1125    run.openedFits.clear();
     1126    return 0;
     1127}
     1128// --------------------------------------------------------------------------
     1129//
     1130//! Add a new active run number
     1131//! @param newRun the new run number
     1132//! @param time the time at which the new run number was issued
     1133//
     1134void DataLogger::AddNewRunNumber(int newRun, Time time)
     1135{
     1136    if (fDebugIsOn)
     1137    {
     1138        stringstream str;
     1139        str << "Adding new run number " << newRun << " that was issued on " << time;
     1140        Debug(str.str());
     1141    }
     1142    //Add new run number to run number list
     1143    fRunNumber.push_back(RunNumberType());
     1144    fRunNumber.back().runNumber = newRun;
     1145    fRunNumber.back().time = time;
     1146    if (GetCurrentState() != kSM_Logging)
     1147        return;
     1148    //open the log and report files
     1149    OpenRunFile(fRunNumber.back());
     1150}
    8601151// --------------------------------------------------------------------------
    8611152//
     
    8691160    if (strstr(I->getName(), fRunNumberInfo) != NULL)
    8701161    {//assumes that the run number is an integer
    871         fRunNumber = I->getInt();   
     1162        AddNewRunNumber(I->getInt(), Time(I->getTimestamp(), I->getTimestampMillisecs()*1000));
    8721163        ostringstream str;
    873         str << "New run number is " << fRunNumber;
     1164        str << "New run number is " << fRunNumber.back().runNumber;
    8741165        Message(str.str());
    8751166    }
     
    9121203        }
    9131204    }
    914        
    9151205    //construct the header
    9161206    ostringstream header;
     
    9181208    fQuality = I->getQuality();
    9191209    fMjD = cTime.Mjd();
     1210
     1211    //figure out which run file should be used
     1212    ofstream* targetRunFile = NULL;
     1213    RunNumberType* cRunNumber = NULL;
     1214    if (GetCurrentState() == kSM_Logging)
     1215    {
     1216        list<RunNumberType>::reverse_iterator rit;
     1217        for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
     1218        {
     1219            if (rit->time < cTime) //this is the run number that we want to use
     1220            {
     1221                //Find something better to convert iterator to pointer than the ugly line below....
     1222                cRunNumber = &(*rit);
     1223                sub.runNumber = rit->runNumber;
     1224                targetRunFile = isItaReport ? rit->reportFile : rit->logFile;
     1225                break;
     1226            }
     1227        }
     1228        if (rit == fRunNumber.rend() && fRunNumber.size() != 0)
     1229        {
     1230            stringstream str;
     1231            str << "Could not find an appropriate run number for info coming at time: " << cTime;
     1232            Error(str.str());
     1233            Error("Active run numbers: ");
     1234            for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
     1235            {
     1236                str.str("");
     1237                str << rit->runNumber;
     1238                Error(str.str());
     1239            }
     1240        }
     1241    }
    9201242
    9211243    if (isItaReport)
     
    9721294        }
    9731295        //write entry to run-report
    974         if (fRunReportFile.is_open())
     1296        if (targetRunFile && targetRunFile->is_open())
    9751297        {
    9761298            if (fDebugIsOn)
     
    9801302                Debug(str.str());   
    9811303            }
    982             fRunReportFile << header.str() << text << endl;
    983             if (!fRunReportFile.good())
     1304            *targetRunFile << header.str() << text << endl;
     1305            if (!targetRunFile->good())
    9841306            {
    9851307                Error("An error occured while writing to the run report file. Closing it.");
    986                 if (fRunReportFile.is_open())
    987                     fRunReportFile.close();   
     1308                if (targetRunFile->is_open())
     1309                    targetRunFile->close();
    9881310            }
    9891311        }
     
    9931315        string n = I->getName();
    9941316        ostringstream msg;
    995         msg << n << ": " << I->getString();//n.substr(0, n.find_first_of('/')) << ": " << I->getString();
     1317        msg << n << ": " << I->getString();
    9961318
    9971319        if (fNightlyLogFile.is_open())
     
    10121334            }
    10131335        }
    1014         if (fRunLogFile.is_open())
     1336        if (targetRunFile && targetRunFile->is_open())
    10151337        {
    10161338            if (fDebugIsOn)
     
    10201342                Debug(str.str());   
    10211343            }
    1022             MessageImp runMess(fRunLogFile);
     1344            MessageImp runMess(*targetRunFile);
    10231345            runMess.Write(cTime, msg.str().c_str(), fQuality);
    1024             if (!fRunLogFile.good())
     1346            if (!targetRunFile->good())
    10251347            {
    10261348                Error("An error occured while writing to the run log file. Closing it.");
    1027                 if (fRunLogFile.is_open())
    1028                     fRunLogFile.close();   
     1349                if (targetRunFile->is_open())
     1350                    targetRunFile->close();
    10291351            }
    10301352        }
     
    10341356    if (isItaReport)
    10351357    {
    1036         if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen())
    1037             OpenFITSFilesPlease(sub);   
     1358        if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen() || sub.runNumber != sub.runFile.fRunNumber)
     1359            OpenFITSFilesPlease(sub, cRunNumber);
    10381360        WriteToFITS(sub);
    10391361    }   
     
    10451367//
    10461368//! write messages to logs.
     1369/*
    10471370//! @param evt
    10481371//!        the current event to log
     
    11381461    }
    11391462    return GetCurrentState();
    1140 }
     1463}*/
    11411464// --------------------------------------------------------------------------
    11421465//
     
    11541477    //print the path configuration
    11551478    string actualTargetDir;
    1156     if (fNightlyFileName == ".")
     1479    if (fNightlyFilePath == ".")
    11571480    {
    11581481        char currentPath[FILENAME_MAX];
     
    11611484    }
    11621485    else
    1163         actualTargetDir = fNightlyFileName;
     1486        actualTargetDir = fNightlyFilePath;
    11641487    Message("Nightly Path: " + actualTargetDir);
    1165     if (fRunFileName == ".")
     1488    if (fRunFilePath == ".")
    11661489    {
    11671490        char currentPath[FILENAME_MAX];
     
    11701493    }
    11711494    else
    1172         actualTargetDir = fRunFileName;
     1495        actualTargetDir = fRunFilePath;
    11731496    Message("Run Path: " + actualTargetDir);
    11741497    ostringstream str;
    1175     str << "Run Number: " << fRunNumber;
     1498    str << "Active Run Numbers: ";//<< fRunNumber;
     1499    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
     1500        str << "\n" << it->runNumber;
    11761501    Message(str.str());
    11771502    Message("----------- OPENED FILES ----------------");
     
    11851510    else
    11861511        Message("Nightly Report.....CLOSED");
    1187     if (fRunLogFile.is_open())
    1188         Message("Run Log..............OPEN");
    1189     else
    1190         Message("Run Log............CLOSED");
    1191     if (fRunReportFile.is_open())
    1192         Message("Run Report...........OPEN");
    1193     else
    1194         Message("Run Report.........CLOSED");
     1512    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
     1513    {
     1514        if (it->logFile->is_open())
     1515            Message("Run Log " + it->logName + " is OPEN");
     1516        else
     1517            Message("Run Log " + it->logName + " is CLOSED");
     1518        if (it->reportFile->is_open())
     1519            Message("Run Report " + it->reportName + " is OPEN");
     1520        else
     1521            Message("Run Report " + it->reportName + " CLOSED");
     1522    }
    11951523    bool statWarning = false;
    11961524    DataLoggerStats statVar;
     
    13651693    if (evt.GetText() != NULL)
    13661694    {
    1367         fNightlyFileName = string(evt.GetText());
    1368         Message("New Nightly folder specified: " + fNightlyFileName);
     1695        fNightlyFilePath = string(evt.GetText());
     1696        Message("New Nightly folder specified: " + fNightlyFilePath);
    13691697    }
    13701698    else
     
    13841712    if (evt.GetText() != NULL)
    13851713    {
    1386         fRunFileName = string(evt.GetText());
    1387         Message("New Run folder specified: " + fRunFileName);
     1714        fRunFilePath = string(evt.GetText());
     1715        Message("New Run folder specified: " + fRunFilePath);
    13881716    }
    13891717    else
     
    14011729int DataLogger::ConfigureRunNumber(const Event& evt)
    14021730{
    1403     fRunNumber = evt.GetInt();
     1731    AddNewRunNumber(evt.GetInt(), evt.GetTime());
     1732//    fRunNumber = evt.GetInt();
    14041733    ostringstream str;
    1405     str << "The new run number is: " << fRunNumber;
     1734    str << "The new run number is: " << fRunNumber.back().runNumber;
    14061735    Message(str.str());
    14071736    return GetCurrentState();
     
    14521781    sTime << time.Y() << "_" << time.M() << "_" << time.D();
    14531782
    1454     fFullNightlyLogFileName = fNightlyFileName + '/' + sTime.str() + ".log";
     1783    fFullNightlyLogFileName = fNightlyFilePath + '/' + sTime.str() + ".log";
    14551784    fNightlyLogFile.open(fFullNightlyLogFileName.c_str(), ios_base::out | ios_base::app);
    14561785    if (errno != 0)
     
    14601789        Error(str);   
    14611790    }
    1462     fFullNightlyReportFileName = fNightlyFileName + '/' + sTime.str() + ".rep";
     1791    fFullNightlyReportFileName = fNightlyFilePath + '/' + sTime.str() + ".rep";
    14631792    fNightlyReportFile.open(fFullNightlyReportFileName.c_str(), ios_base::out | ios_base::app);
    14641793    if (errno != 0)
     
    14871816    //notify that files were opened
    14881817    string actualTargetDir;
    1489     if (fNightlyFileName == ".")
     1818    if (fNightlyFilePath == ".")
    14901819    {
    14911820        char currentPath[FILENAME_MAX];
     
    15031832    else
    15041833    {
    1505         actualTargetDir = fNightlyFileName;   
     1834        actualTargetDir = fNightlyFilePath;
    15061835    }
    15071836    //notify that a new file has been opened.
     
    15191848//! @param sub
    15201849//!     the current DimInfo subscription being examined
    1521 void DataLogger::OpenFITSFilesPlease(SubscriptionType& sub)
     1850void DataLogger::OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber)
    15221851{
    15231852    string serviceName(sub.dimInfo->getName());
    1524     //we must check if we should group this service subscription to a single fits file before we replace the / by _
     1853    //if run number has changed, reopen a new fits file with the correct run number.
     1854     if (sub.runFile.IsOpen() && sub.runFile.fRunNumber != sub.runNumber)
     1855     {
     1856         if (fDebugIsOn)
     1857         {
     1858             Debug("Run number changed. Closing " + sub.runFile.fFileName);
     1859         }
     1860         sub.runFile.Close();
     1861     }
     1862      //we must check if we should group this service subscription to a single fits file before we replace the / by _
    15251863    bool hasGrouping = false;
    15261864    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
     
    15351873        }
    15361874    }
     1875    hasGrouping = true;
    15371876    for (unsigned int i=0;i<serviceName.size(); i++)
    15381877    {
     
    15501889    {
    15511890        string fileNameOnly = sTime.str() + '_' + serviceName + ".fits";
    1552         string partialName = fNightlyFileName + '/' + fileNameOnly;
     1891        string partialName = fNightlyFilePath + '/' + fileNameOnly;
    15531892        AllocateFITSBuffers(sub);
    15541893        //get the size of the file we're about to open
     
    15621901            fOpenedNightlyFits[fileNameOnly].push_back(serviceName);
    15631902        }
    1564         sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out());
     1903        sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, -1);//Out());
    15651904
    15661905        //notify the opening
    15671906        string actualTargetDir;
    1568         if (fNightlyFileName == ".")
     1907        if (fNightlyFilePath == ".")
    15691908        {
    15701909            char currentPath[FILENAME_MAX];
     
    15741913        else
    15751914        {
    1576             actualTargetDir = fNightlyFileName;   
     1915            actualTargetDir = fNightlyFilePath;
    15771916        }       
    15781917        NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 7, fOpenedNightlyFiles);
     
    15861925        }
    15871926    }
    1588     if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
     1927    //to the actual file open
     1928    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging) && sub.runNumber != -1)
    15891929    {//buffer for the run file have already been allocated when doing the Nightly file
    15901930        ostringstream sRun;
    1591         sRun << fRunNumber;
     1931        sRun << sub.runNumber;
    15921932        string fileNameOnly;
    15931933        string partialName;
     
    15961936        {
    15971937            fileNameOnly = sRun.str() + "_group.fits";
    1598             partialName = fRunFileName + '/' + fileNameOnly;
     1938            partialName = fRunFilePath + '/' + fileNameOnly;
    15991939        }
    16001940        else
    16011941        {
    16021942            fileNameOnly = sRun.str() + '_' + serviceName + ".fits";
    1603             partialName = fRunFileName + '/' + fileNameOnly;
     1943            partialName = fRunFilePath + '/' + fileNameOnly;
    16041944        }
    16051945        //get the size of the file we're about to open
     
    16121952                fBaseSizeRun = 0;
    16131953            fFileSizesMap[partialName] = 0;
    1614             fOpenedRunFits[fileNameOnly].push_back(serviceName);
     1954            cRunNumber->openedFits[fileNameOnly].push_back(serviceName);
    16151955        }
    16161956        else
     
    16201960             //and reopened again. Unlikely to happen, but well it may
    16211961                bool found = false;
    1622                 for (vector<string>::iterator it=fOpenedRunFits[fileNameOnly].begin(); it!=fOpenedRunFits[fileNameOnly].end(); it++)
     1962                for (vector<string>::iterator it=cRunNumber->openedFits[fileNameOnly].begin(); it!=cRunNumber->openedFits[fileNameOnly].end(); it++)
    16231963                    if (*it == serviceName)
    16241964                    {
     
    16271967                    }
    16281968                if (!found)
    1629                     fOpenedRunFits[fileNameOnly].push_back(serviceName);
    1630             }
    1631         if (hasGrouping && fRunFitsFile == NULL)
     1969                    cRunNumber->openedFits[fileNameOnly].push_back(serviceName);
     1970            }
     1971
     1972        if (hasGrouping && cRunNumber->runFitsFile == NULL)
    16321973            try
    16331974            {
    1634                 fRunFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write);
     1975                cRunNumber->runFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write);
    16351976                (fNumSubAndFitsData.numOpenFits)++;
    16361977            }   
     
    16401981                str << "Could not open FITS Run file " << partialName << " reason: " << e.message();
    16411982                Error(str);
    1642                 fRunFitsFile = NULL;
     1983                cRunNumber->runFitsFile = NULL;
    16431984            }
    16441985
    16451986        string actualTargetDir;
    1646         if (fRunFileName == ".")
     1987        if (fRunFilePath == ".")
    16471988        {
    16481989            char currentPath[FILENAME_MAX];
     
    16521993        else
    16531994        {
    1654            actualTargetDir = fRunFileName;
     1995           actualTargetDir = fRunFilePath;
    16551996        }
    16561997        NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 7, fOpenedRunFiles);// + '_' + serviceName, 4);
    16571998        if (hasGrouping)
    1658             sub.runFile.Open(partialName, serviceName, fRunFitsFile, &fNumSubAndFitsData.numOpenFits, this);//Out());
     1999            sub.runFile.Open(partialName, serviceName, cRunNumber->runFitsFile, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber);//Out());
    16592000        else
    1660             sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out());
     2001            sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber);//Out());
    16612002
    16622003       if (fNumSubAndFitsIsOn)
     
    16712012}   
    16722013// --------------------------------------------------------------------------
     2014//
     2015//! Allocates the required memory for a given pair of fits files (nightly and run)
     2016//! @param sub the subscription of interest.
    16732017//
    16742018void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
     
    17852129        Debug("Starting Run Logging...");   
    17862130    }
    1787     //attempt to open run file with current parameters
    1788 //    if (fRunNumber == -1)
    1789 //        return kSM_BadRunConfig;
    1790     ostringstream sRun;
    1791     sRun << fRunNumber;
    1792     fFullRunLogFileName = fRunFileName + '/' + sRun.str() + ".log";
    1793     fRunLogFile.open(fFullRunLogFileName.c_str(), ios_base::out | ios_base::app); //maybe should be app instead of ate
    1794     if (errno != 0)
    1795     {
    1796         ostringstream str;
    1797         str << "Unable to open run Log " << fFullRunLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
    1798         Error(str);   
    1799     }
    1800     fFullRunReportFileName = fRunFileName + '/' + sRun.str() + ".rep";
    1801     fRunReportFile.open(fFullRunReportFileName.c_str(), ios_base::out | ios_base::app);
    1802     if (errno != 0)
    1803     {
    1804         ostringstream str;
    1805         str << "Unable to open run report " << fFullRunReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
    1806         Error(str);   
    1807     }
    1808    
    1809     if (!fRunLogFile.is_open() || !fRunReportFile.is_open())
    1810     {
    1811         ostringstream str;
    1812         str << "Something went wrong while openning nightly files " << fFullRunLogFileName << " and " << fFullRunReportFileName;
    1813         Error(str.str());
    1814         return kSM_BadRunConfig;
    1815     }
    1816     //get the size of the newly opened file.
    1817     struct stat st;
    1818     fBaseSizeRun = 0;
    1819     if (fFileSizesMap.find(fFullRunLogFileName) == fFileSizesMap.end())
    1820     {
    1821         stat(fFullRunLogFileName.c_str(), &st);
    1822         if (errno != 0)
    1823         {
    1824             ostringstream str;
    1825             str << "Unable to stat " << fFullRunLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
    1826             Error(str);   
    1827         }
    1828         else
    1829             fBaseSizeRun += st.st_size;
    1830         fFileSizesMap[fFullRunLogFileName] = 0;
    1831     }
    1832     if (fFileSizesMap.find(fFullRunReportFileName) == fFileSizesMap.end())
    1833     {
    1834         stat(fFullRunReportFileName.c_str(), &st);
    1835         if (errno != 0)
    1836         {
    1837             ostringstream str;
    1838             str << "Unable to stat " << fFullRunReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
    1839             Error(str);   
    1840         }
    1841         else
    1842             fBaseSizeRun += st.st_size;
    1843         fFileSizesMap[fFullRunReportFileName] = 0;
    1844     }
    1845     string actualTargetDir;
    1846     if (fRunFileName == ".")
    1847     {
    1848         char currentPath[FILENAME_MAX];
    1849         if (!getcwd(currentPath, sizeof(currentPath)))
    1850         {
    1851             if (errno != 0)
    1852             {
    1853                 ostringstream str;
    1854                 str << "Unable to retrieve the current path" << ". Reason: " << strerror(errno) << " [" << errno << "]";
    1855                 Error(str);
    1856             }
    1857         }
    1858         actualTargetDir = currentPath;
    1859     }
    1860     else
    1861     {
    1862         actualTargetDir = fRunFileName;   
    1863     }       
    1864     NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 3, fOpenedRunFiles);
    1865    
    1866     fOpenedRunFits.clear();
     2131    //open all the relevant run-files. i.e. all the files associated with run numbers.
     2132    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
     2133        OpenRunFile(*it);
    18672134
    18682135    return kSM_Logging;
    18692136}
     2137
    18702138#ifdef HAVE_FITS
    1871 void DataLogger::CreateFitsGrouping(bool runGroup)
     2139// --------------------------------------------------------------------------
     2140//
     2141//! Create a fits group file with all the run-fits that were written (either daily or run)
     2142//! @param filesToGroup a map of filenames mapping to table names to be grouped (i.e. a
     2143//!        single file can contain several tables to group
     2144//! @param runNumber the run number that should be used for grouping. -1 means nightly group
     2145//
     2146void DataLogger::CreateFitsGrouping(map<string, vector<string> > & filesToGroup, int runNumber)
    18722147{
    18732148    if (fDebugIsOn)
     
    18752150        ostringstream str;
    18762151        str << "Creating fits group for ";
    1877         if (runGroup)
     2152        if (runNumber != -1)
    18782153            str << "run files";
    18792154        else
     
    18832158    //create the FITS group corresponding to the ending run.
    18842159    CCfits::FITS* groupFile;
    1885     map<string, vector<string> >& filesToGroup = runGroup? fOpenedRunFits : fOpenedNightlyFits;
    18862160    unsigned int numFilesToGroup = 0;
    18872161    for (map<string, vector<string> >::iterator it=filesToGroup.begin(); it != filesToGroup.end(); it++)
     
    19012175    }
    19022176    ostringstream groupName;
    1903     if (runGroup)
    1904     {
    1905         groupName << fRunFileName << '/' << fRunNumber << ".fits";
     2177    if (runNumber != -1)
     2178    {
     2179        groupName << fRunFilePath << '/' << runNumber << ".fits";
    19062180    }
    19072181    else
     
    19112185        sTime << time.Y() << "_" << time.M() << "_" << time.D();
    19122186
    1913         groupName << fNightlyFileName << '/' << sTime.str() << ".fits";
     2187        groupName << fNightlyFilePath << '/' << sTime.str() << ".fits";
    19142188    }
    19152189    CCfits::Table* groupTable;
     
    19332207
    19342208        groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
     2209//TODO handle the case when the logger was stopped and restarted during the same day, i.e. the grouping file must be updated
    19352210     }
    19362211     catch (CCfits::FitsException e)
     
    19522227    char* startOfLocation = reinterpret_cast<char*>(&fitsBuffer[8 + 3]);
    19532228    char* startOfName = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]);
    1954  //   char* startOfNameVisible = reinterpret_cast<char*>(&fitsBuffer[8 + 3 + 2*maxCharLength]);
     2229
    19552230    sprintf(startOfExtension, "%s", "BINTABLE");
    19562231    sprintf(startOfURI, "%s", "URL");
     
    19672242                Debug(str.str());
    19682243            }
    1969  //           strcpy(startOfNameVisible, jt->c_str());
    19702244            int status = 0;
    19712245            fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status);
     
    19822256}
    19832257#endif //HAVE_FITS
     2258
    19842259// --------------------------------------------------------------------------
    19852260//
     
    19952270        Debug("Stopping Run Logging...");   
    19962271    }
    1997     if (!fRunLogFile.is_open() || !fRunReportFile.is_open())
    1998         return kSM_FatalError;
    1999     if (fRunLogFile.is_open())
    2000         fRunLogFile.close();
    2001     if (fRunReportFile.is_open())
    2002         fRunReportFile.close();
     2272    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
     2273    {
     2274        if (!it->logFile->is_open() || !it->reportFile->is_open())
     2275            return kSM_FatalError;
     2276        it->logFile->close();
     2277        it->reportFile->close();
     2278    }
     2279
    20032280#ifdef HAVE_FITS
    20042281    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
     
    20082285                j->second.runFile.Close();
    20092286        }
    2010     if (fRunFitsFile != NULL)
    2011     {
    2012         delete fRunFitsFile;
    2013         fRunFitsFile = NULL;   
    2014         (fNumSubAndFitsData.numOpenFits)--;
    2015     }
    20162287#endif
    20172288    NotifyOpenedFile("", 0, fOpenedRunFiles);
     
    20192290        fNumSubAndFits->updateService();
    20202291
    2021     CreateFitsGrouping(true);
     2292    while (fRunNumber.size() > 0)
     2293    {
     2294        RemoveOldestRunNumber();
     2295    }
    20222296
    20232297    return kSM_WaitingRun;
    2024 
    20252298}
    20262299// --------------------------------------------------------------------------
     
    20322305int DataLogger::GoToReadyPlease()
    20332306{
    2034     if (fDebugIsOn)
     2307   if (fDebugIsOn)
    20352308    {
    20362309        Debug("Going to the Ready state...");
    20372310    }   
     2311   if (GetCurrentState() == kSM_Logging)
     2312       StopRunPlease();
     2313
    20382314    if (fNightlyLogFile.is_open())
    20392315        fNightlyLogFile.close();
    20402316    if (fNightlyReportFile.is_open())
    20412317        fNightlyReportFile.close();
    2042 
    2043     if (fRunLogFile.is_open())
    2044         fRunLogFile.close();
    2045     if (fRunReportFile.is_open())
    2046         fRunReportFile.close();
    20472318       
    20482319#ifdef HAVE_FITS
     
    20512322        {
    20522323            if (j->second.nightlyFile.IsOpen())
    2053                 j->second.nightlyFile.Close();
    2054             if (j->second.runFile.IsOpen())
    2055                 j->second.runFile.Close();
    2056         }
    2057     if (fRunFitsFile != NULL)
    2058     {
    2059         delete fRunFitsFile;
    2060         fRunFitsFile = NULL;
    2061         (fNumSubAndFitsData.numOpenFits)--;
    2062     }
     2324                j->second.nightlyFile.Close();;
     2325        }
    20632326#endif
    2064     if (GetCurrentState() == kSM_Logging)
    2065         NotifyOpenedFile("", 0, fOpenedRunFiles);
    20662327    if (GetCurrentState() == kSM_Logging ||
    20672328        GetCurrentState() == kSM_WaitingRun ||
     
    20722333            fNumSubAndFits->updateService();
    20732334    }
    2074     CreateFitsGrouping(true);
    2075     CreateFitsGrouping(false);
    2076 
     2335#ifdef HAVE_FITS
     2336    CreateFitsGrouping(fOpenedNightlyFits, -1);
     2337#endif
    20772338    return kSM_Ready;
    20782339}
     
    20912352    return kSM_WaitingRun;   
    20922353}
    2093 
     2354// --------------------------------------------------------------------------
     2355//
     2356//! Setup Logger's configuration from a Configuration object
     2357//! @param conf the configuration object that should be used
     2358//!
    20942359bool DataLogger::SetConfiguration(Configuration& conf)
    20952360{
     
    21492414
    21502415// --------------------------------------------------------------------------
    2151 
    21522416int RunDim(Configuration &conf)
    21532417{
     
    21692433    return 0;
    21702434}
    2171 
     2435// --------------------------------------------------------------------------
    21722436void RunThread(DataLogger* logger)
    21732437{
     
    21772441    Readline::Stop();   
    21782442}
    2179 
     2443// --------------------------------------------------------------------------
    21802444template<class T>
    21812445int RunShell(Configuration &conf)
     
    22352499
    22362500}
    2237 
     2501// --------------------------------------------------------------------------
    22382502void PrintHelp()
    22392503{
     
    22682532    cout << endl;
    22692533}
    2270 
     2534// --------------------------------------------------------------------------
    22712535void SetupConfiguration(Configuration &conf)
    22722536{
     
    22932557    conf.AddOptions(configs);
    22942558}
    2295 
     2559// --------------------------------------------------------------------------
    22962560int main(int argc, const char* argv[])
    22972561{
Note: See TracChangeset for help on using the changeset viewer.