source: trunk/FACT++/src/dataLogger.cc @ 10741

Last change on this file since 10741 was 10741, checked in by lyard, 9 years ago
added configurable grouping and did some cosmetics to the code (removed tabs)
File size: 83.2 KB
Line 
1//****************************************************************
2/** @class DataLogger
3 
4  @brief Logs all message and infos between the services
5 
6  This is the main logging class facility.
7  It derives from StateMachineDim and DimInfoHandler. the first parent is here to enforce
8  a state machine behaviour, while the second one is meant to make the dataLogger receive
9  dim services to which it subscribed from.
10  The possible states and transitions of the machine are:
11  \dot
12  digraph datalogger {
13          node [shape=record, fontname=Helvetica, fontsize=10];
14      e [label="Error" color="red"];
15   r [label="Ready"]
16   d [label="NightlyOpen"]
17   w [label="WaitingRun"]
18      l [label="Logging"]
19   b [label="BadNightlyconfig" color="red"]
20   c [label="BadRunConfig" color="red"]
21 
22  e -> r
23  r -> e
24  r -> d
25  r -> b
26  d -> w
27  d -> r
28  w -> r
29  l -> r
30  l -> w
31  b -> d
32  w -> c
33  w -> l
34  b -> r
35  c -> r
36  c -> l
37  }
38  \enddot
39 */
40 //****************************************************************
41#include "FACT.h"
42#include "Dim.h"
43#include "Event.h"
44#include "Time.h"
45#include "StateMachineDim.h"
46#include "WindowLog.h"
47#include "Configuration.h"
48#include "ServiceList.h"
49#include "Converter.h"
50#include "MessageImp.h"
51#include "LocalControl.h"
52#include "DimDescriptionService.h"
53
54#include "Description.h"
55
56#include "DimServiceInfoList.h"
57
58//for getting stat of opened files
59#include <unistd.h>
60//for getting disk free space
61#include <sys/statvfs.h>
62//for getting files sizes
63#include <sys/stat.h>
64
65#include <fstream>
66
67#include <boost/bind.hpp>
68#if BOOST_VERSION < 104400
69#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
70#undef BOOST_HAS_RVALUE_REFS
71#endif
72#endif
73#include <boost/thread.hpp>
74
75#ifdef HAVE_FITS
76#include "Fits.h"
77#endif
78
79//Dim structures
80struct DataLoggerStats {
81    long sizeWritten;
82    long freeSpace;
83    long writingRate;
84};
85
86struct NumSubAndFitsType {
87    int numSubscriptions;
88    int numOpenFits;
89};
90
91struct OpenFileToDim {
92    int code;
93    char fileName[FILENAME_MAX];
94};
95
96class DataLogger : public StateMachineDim, DimInfoHandler
97{
98public:
99    /// The list of existing states specific to the DataLogger
100    enum
101    {
102        kSM_NightlyOpen = 20, ///< Nightly file openned and writing
103        kSM_WaitingRun = 30, ///< waiting for the run number to open the run file
104        kSM_Logging = 40, ///< both files openned and writing
105        kSM_BadNightlyConfig = 0x101, ///< the folder specified for Nightly logging does not exist or has bad permissions
106        kSM_BadRunConfig = 0x102, ///<  the folder specified for the run logging does not exist or has wrong permissions or no run number
107    } localstates_t;
108   
109    DataLogger(ostream &out);
110    ~DataLogger(); 
111   
112private:
113    //Define all the data structure specific to the DataLogger here
114     /// ofstream for the NightlyLogfile
115    ofstream fNightlyLogFile;
116    /// ofstream for the run-specific Log file
117    ofstream fRunLogFile;
118
119    /// ofstream for the Nightly report file
120    ofstream fNightlyReportFile;
121    /// ofstream for the run-specific report file
122    ofstream fRunReportFile;
123    /// base path of the Nightlyfile
124    string fNightlyFileName;
125    ///base path of the run file
126    string fRunFileName;
127    ///run number (-1 means no run number specified)
128    int fRunNumber; 
129    ///previous run number. to check if changed while logging
130    int fPreviousRunNumber;
131    ///Current Service Quality
132    int fQuality;
133    ///Modified Julian Date
134    double fMjD;
135public:
136    ///Define all the static names
137    static const char* fConfigDay;
138    static const char* fConfigRun;
139    static const char* fConfigRunNumber;
140    static const char* fConfigLog;
141    static const char* fTransStart;
142    static const char* fTransStop;
143    static const char* fTransStartRun;
144    static const char* fTransStopRun;
145    static const char* fTransReset;
146    static const char* fTransWait;
147    static const char* fRunNumberInfo; ///< This is the name of the dimInfo received to specify the run number. It must be updated once the final name will be defined
148    static const char* fPrintCommand;
149    static const char* fDebugOnOff;
150    static const char* fStatsPeriod;
151    static const char* fStartStopOpenedFiles;
152    static const char* fStartStopNumSubsAndFits;
153private:
154    //overloading of DIM's infoHandler function
155    void infoHandler(); 
156   
157    ///for obtaining the name of the existing services
158    ServiceList fServiceList;
159   
160    ///A std pair to store both the DimInfo pointer and the corresponding outputted fits file
161    struct SubscriptionType
162    { 
163#ifdef HAVE_FITS
164        ///Nightly FITS output file
165        Fits    nightlyFile;
166        ///run-specific FITS output file
167        Fits    runFile;
168#endif
169        ///the actual dimInfo pointer
170        DimStampedInfo* dimInfo;
171        ///the converter for outputting the data according to the format
172        Converter* fConv;
173        ///the number of existing handlers to this structure.
174        ///This is required otherwise I MUST handle the deleting of dimInfo outside from the destructor
175        int* numCopies;
176        void operator = (const SubscriptionType& other)
177        {
178#ifdef HAVE_FITS
179            nightlyFile = other.nightlyFile;
180            runFile = other.runFile;
181#endif
182            dimInfo = other.dimInfo;   
183            numCopies = other.numCopies;
184            fConv = other.fConv;
185            (*numCopies)++;
186        }
187        SubscriptionType(const SubscriptionType& other)
188        {
189#ifdef HAVE_FITS
190            nightlyFile = other.nightlyFile;
191            runFile = other.runFile;
192#endif
193            dimInfo = other.dimInfo;
194            numCopies = other.numCopies;
195            fConv = other.fConv;
196            (*numCopies)++;   
197        }
198        SubscriptionType(DimStampedInfo* info)
199        {
200            dimInfo = info;   
201            fConv = NULL;
202            numCopies = new int(1);
203        }
204        SubscriptionType()
205        {
206            dimInfo = NULL;
207            fConv = NULL;
208            numCopies = new int(1);
209        }
210        ~SubscriptionType()
211        {
212            if (numCopies)
213            (*numCopies)--;
214            if (numCopies)
215            if (*numCopies < 1)
216            {
217                if (dimInfo)
218                delete dimInfo;
219#ifdef HAVE_FITS
220                if (nightlyFile.IsOpen())
221                    nightlyFile.Close();
222                if (runFile.IsOpen())
223                    runFile.Close();
224#endif
225                if (numCopies)   
226                delete numCopies;
227                delete fConv;
228                fConv = NULL;
229                dimInfo = NULL;
230                numCopies = NULL;
231            }
232        }
233    };
234    typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
235    ///All the services to which we have subscribed to, sorted by server name.
236    SubscriptionsListType fServiceSubscriptions;
237
238    ///Reporting method for the services info received
239    void ReportPlease(DimInfo* I, SubscriptionType& sub); 
240
241    ///Configuration of the nightly file path
242    int ConfigureNightlyFileName(const Event& evt); 
243    ///Configuration fo the file name
244    int ConfigureRunFileName(const Event& evt); 
245    ///DEPREC - configuration of the run number
246    int ConfigureRunNumber(const Event& evt); 
247    ///logging method for the messages
248    int LogMessagePlease(const Event& evt); 
249    ///print the current state of the dataLogger
250    int PrintStatePlease(const Event& evt);
251    ///checks whether or not the current info being treated is a run number
252    void CheckForRunNumber(DimInfo* I); 
253    /// start transition
254    int StartPlease(); 
255    ///from waiting to logging transition
256    int StartRunPlease(); 
257    /// from logging to waiting transition
258    int StopRunPlease(); 
259    ///stop and reset transition
260    int GoToReadyPlease(); 
261    ///from NightlyOpen to waiting transition
262    int NightlyToWaitRunPlease(); 
263#ifdef HAVE_FITS
264    ///Open fits files
265    void OpenFITSFilesPlease(SubscriptionType& sub);
266    ///Write data to FITS files
267    void WriteToFITS(SubscriptionType& sub);
268    ///Allocate the buffers required for fits
269    void AllocateFITSBuffers(SubscriptionType& sub);
270    ///FITS file for runs grouping. only one, hence dealt with in the dataLogger itself
271    CCfits::FITS* fRunFitsFile;
272#endif//has_fits
273public:   
274    ///checks with fServiceList whether or not the services got updated
275    bool CheckForServicesUpdate(); 
276
277private:   
278    ///monitoring notification loop
279    void ServicesMonitoring();
280    ///services notification thread
281    boost::thread fMonitoringThread;
282    ///end of the monitoring
283    bool fContinueMonitoring;
284    ///required for accurate monitoring
285    map<string, long> fFileSizesMap;
286    string fFullNightlyLogFileName;
287    string fFullNightlyReportFileName;
288    string fFullRunLogFileName;
289    string fFullRunReportFileName;
290    long fBaseSizeNightly;
291    long fPreviousSize;
292    long fBaseSizeRun;
293    ///Service for opened files
294    DimDescribedService* fOpenedNightlyFiles;
295    DimDescribedService* fOpenedRunFiles;
296    DimDescribedService* fNumSubAndFits;
297    NumSubAndFitsType fNumSubAndFitsData;
298
299    inline void NotifyOpenedFile(string name, int type, DimDescribedService* service);
300
301public:   
302    bool SetConfiguration(Configuration& conf);
303
304private:
305    set<string> fBlackList;
306    set<string> fWhiteList;
307    set<string> fGrouping;
308    bool fHasBlackList;
309    bool fHasWhiteList;
310    bool fDebugIsOn;
311    float fStatsPeriodDuration;
312    bool fOpenedFilesIsOn;
313    bool fNumSubAndFitsIsOn;
314    //functions for controlling the services behavior
315    int SetDebugOnOff(const Event& evt);
316    int SetStatsPeriod(const Event& evt);
317    int SetOpenedFilesOnOff(const Event& evt);
318    int SetNumSubsAndFitsOnOff(const Event& evt);
319    ///boolean to prevent DIM update while desctructing the dataLogger
320    bool fDestructing;   
321
322    ///Small function for calculating the total size written so far
323    void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting);
324
325    ///vectors to keep track of opened Fits files, for grouping purposes.
326    //This cannot be done otherwise, as services may disapear before the files are nicely closed. Hence which files were opened must be remembered.
327    map<string, vector<string> > fOpenedRunFits;
328    map<string, vector<string> > fOpenedNightlyFits;
329    void CreateFitsGrouping(bool runGroup);
330}; //DataLogger
331
332void DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting)
333{
334#ifdef HAVE_FITS
335    if (isPrinting)
336    {
337        stringstream str;
338        str.str("");
339        str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:";
340        Message(str.str());
341    }
342    SubscriptionsListType::iterator x;
343    map<string, SubscriptionType>::iterator y;
344///TODO the grouping file is dealt with several times. This should not be a problem but well, better to fix it I guess.
345    for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
346    {
347        for (y=x->second.begin(); y != x->second.end(); y++)
348        {
349            if (y->second.runFile.IsOpen())
350            {
351                    fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();
352                    if (isPrinting)
353                        Message("-> "+y->second.runFile.fFileName);
354            }
355            if (y->second.nightlyFile.IsOpen())
356            {
357                fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();
358                if (isPrinting)
359                    Message("-> "+y->second.nightlyFile.fFileName);
360            }
361        }
362    }
363#else
364    if (isPrinting)
365        Message("FITS output disabled at compilation");
366#endif
367    struct stat st;
368    //gather log and report files sizes on disk
369    if (fNightlyLogFile.is_open())
370    {
371        stat(fFullNightlyLogFileName.c_str(), &st);
372        fFileSizesMap[fFullNightlyLogFileName] = st.st_size;
373    }
374    if (fNightlyReportFile.is_open())
375    {
376        stat(fFullNightlyReportFileName.c_str(), &st);
377        fFileSizesMap[fFullNightlyReportFileName] = st.st_size;
378    }
379    if (fRunLogFile.is_open())
380    {
381        stat(fFullRunLogFileName.c_str(), &st);
382        fFileSizesMap[fFullRunLogFileName] = st.st_size;
383    }
384    if (fRunReportFile.is_open())
385    {
386        stat(fFullRunReportFileName.c_str(), &st);
387        fFileSizesMap[fFullRunReportFileName] = st.st_size;
388    }
389    struct statvfs vfs;
390    if (!statvfs(fNightlyFileName.c_str(), &vfs))
391    {
392        statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
393        shouldWarn = false;
394    }
395    else
396    {
397        stringstream str;
398        str.str("");
399        str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
400        if (!shouldWarn)
401            Error(str);
402        shouldWarn = true;
403        statVar.freeSpace = -1;
404    }
405    //sum up all the file sizes. past and present
406    statVar.sizeWritten = 0;
407    for (map<string, long>::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end();  it++)
408        statVar.sizeWritten += it->second;
409    statVar.sizeWritten -= fBaseSizeNightly;
410    statVar.sizeWritten -= fBaseSizeRun;
411}
412//static members initialization
413//since I do not check the transition/config names any longer, indeed maybe these could be hard-coded... but who knows what will happen in the future ?
414const char* DataLogger::fConfigDay = "CONFIG_DAY";
415const char* DataLogger::fConfigRun = "CONFIG_RUN";
416const char* DataLogger::fConfigRunNumber = "CONFIG_RUN_NUMBER";
417const char* DataLogger::fConfigLog = "LOG";
418const char* DataLogger::fTransStart = "START";
419const char* DataLogger::fTransStop = "STOP";
420const char* DataLogger::fTransStartRun = "START_RUN";
421const char* DataLogger::fTransStopRun = "STOP_RUN";
422const char* DataLogger::fTransReset = "RESET";
423const char* DataLogger::fTransWait = "WAIT_RUN_NUMBER";
424const char* DataLogger::fRunNumberInfo = "RUN_NUMBER";
425const char* DataLogger::fPrintCommand = "PRINT";
426const char* DataLogger::fDebugOnOff = "DEBUG";
427const char* DataLogger::fStatsPeriod = "STATS_PERIOD";
428const char* DataLogger::fStartStopOpenedFiles = "OPENED_FILES_SRVC";
429const char* DataLogger::fStartStopNumSubsAndFits = "NUM_SUBS_SRVC";
430
431void DataLogger::ServicesMonitoring()
432{
433        DataLoggerStats statVar;
434        statVar.sizeWritten = 0;
435        statVar.freeSpace = 0;
436        statVar.writingRate = 0;
437
438        struct statvfs vfs;
439        if (!statvfs(fNightlyFileName.c_str(), &vfs))
440            statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
441        else
442            statVar.freeSpace = -1;
443
444        DimDescribedService srvc ("DATA_LOGGER/STATS", "X:3", statVar, "Add description here");
445        fPreviousSize = 0;
446        bool statWarning = false;
447        //loop-wait for broadcast
448        while (fContinueMonitoring)
449        {
450            if (fStatsPeriodDuration == 0.0f)
451            {
452                sleep(0.1f);
453                continue;
454            }
455            else
456                sleep(fStatsPeriodDuration);
457            //update the fits files sizes
458            calculateTotalSizeWritten(statVar, statWarning, false);
459            if (fStatsPeriodDuration == 0.0f)
460                continue;
461            statVar.writingRate = (statVar.sizeWritten - fPreviousSize)/fStatsPeriodDuration; 
462
463            fPreviousSize = statVar.sizeWritten;
464            if (statVar.writingRate != 0) //if data has been written
465            {
466                srvc.updateService();
467                if(fDebugIsOn)
468                {
469                    stringstream str;
470                    str << "Size written: " << statVar.sizeWritten/1024 << " KB; writting rate: ";
471                    str << statVar.writingRate/1024 << " KB/s; free space: ";
472                    str << statVar.freeSpace/(1024*1024) << " MB";
473                    Debug(str.str());
474                }
475            }
476        }
477}
478
479
480// --------------------------------------------------------------------------
481//
482//! Default constructor. The name of the machine is given DATA_LOGGER
483//! and the state is set to kSM_Ready at the end of the function.
484//
485//!Setup the allows states, configs and transitions for the data logger
486//
487DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER")
488{
489    dic_disable_padding();
490    dis_disable_padding();
491    //initialize member data
492    fNightlyFileName = ".";
493    fRunFileName = ".";
494    fRunNumber = -1;
495    fPreviousRunNumber = fRunNumber;
496#ifdef HAVE_FITS
497    fRunFitsFile = NULL;
498#endif
499
500     //Give a name to this machine's specific states
501     AddStateName(kSM_NightlyOpen,      "NightlyFileOpen",  "The summary files for the night are open.");
502     AddStateName(kSM_WaitingRun,       "WaitForRun",       "The summary files for the night are open and we wait for a run to be started.");
503     AddStateName(kSM_Logging,          "Logging",          "The summary files for the night and the files for a single run are open.");
504     AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "The folder for the nighly summary files is invalid.");
505     AddStateName(kSM_BadRunConfig,     "ErrRunFolder",     "The folder for the run files is invalid.");
506
507     /*Add the possible transitions for this machine*/
508     AddEvent(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig)
509             (boost::bind(&DataLogger::StartPlease, this))
510             ("Start the nightly logging. Nightly file location must be specified already");
511
512     AddEvent(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging)
513             (boost::bind(&DataLogger::GoToReadyPlease, this))
514             ("Stop all data logging, close all files.");
515
516     AddEvent(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig)
517             (boost::bind(&DataLogger::StartRunPlease, this))
518             ("Start the run logging. Run file location must be specified already.");
519
520     AddEvent(kSM_WaitingRun, fTransStopRun, kSM_Logging)
521             (boost::bind(&DataLogger::StopRunPlease, this))
522             ("Wait for a run to be started, open run-files as soon as a run number arrives.");
523
524     AddEvent(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_Error)
525             (boost::bind(&DataLogger::GoToReadyPlease, this))
526             ("Transition to exit error states. Closes the nightly file if already opened.");
527
528     AddEvent(kSM_WaitingRun, fTransWait, kSM_NightlyOpen)
529             (boost::bind(&DataLogger::NightlyToWaitRunPlease, this));
530
531     /*Add the possible configurations for this machine*/
532     AddEvent(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig)
533             (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1))
534             ("Configure the folder for the nightly files."
535              "|Path[string]:Absolute or relative path name where the nightly files should be stored.");
536
537     AddEvent(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
538             (boost::bind(&DataLogger::ConfigureRunFileName, this, _1))
539             ("Configure the folder for the run files."
540              "|Path[string]:Absolute or relative path name where the run files should be stored.");
541
542     AddEvent(fConfigRunNumber, "I", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
543             (boost::bind(&DataLogger::ConfigureRunNumber, this, _1))
544             ("configure the run number. cannot be done in logging state");
545
546     //Provide a logging command
547     //I get the feeling that I should be going through the EventImp
548     //instead of DimCommand directly, mainly because the commandHandler
549     //is already done in StateMachineImp.cc
550     //Thus I'll simply add a configuration, which I will treat as the logging command
551     AddEvent(fConfigLog, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadRunConfig)
552             (boost::bind(&DataLogger::LogMessagePlease, this, _1))
553             ("Log a single message to the log-files."
554              "|Message[string]:Message to be logged.");
555       
556     //Provide a print command
557     stringstream str;
558     str <<     kSM_Ready << " " << kSM_NightlyOpen << " " << kSM_WaitingRun << " " << kSM_Logging << " " << kSM_BadNightlyConfig;
559     str << " " << kSM_BadRunConfig;
560     AddEvent(fPrintCommand, str.str().c_str(), "")
561             (boost::bind(&DataLogger::PrintStatePlease, this, _1))
562             ("Print information about the internal status of the data logger.");
563
564     fServiceList.SetHandler(this);
565     CheckForServicesUpdate();
566
567     //start the monitoring service
568     fContinueMonitoring = true;
569     fMonitoringThread = boost::thread(boost::bind(&DataLogger::ServicesMonitoring, this));
570     fBaseSizeNightly = 0;
571     fBaseSizeRun = 0;
572     OpenFileToDim fToDim;
573     fToDim.code = 0;
574     fToDim.fileName[0] = '\0';
575
576     fOpenedNightlyFiles = new DimDescribedService(GetName() + "/FILENAME_NIGHTLY", "I:1;C", fToDim,
577                               "Path and base name which is used to compile the filenames for the nightly files."
578                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
579                               "|Name[string]:path and base file name");
580
581     fOpenedRunFiles = new DimDescribedService(GetName() + "/FILENAME_RUN", "I:1;C", fToDim,
582                               "Path and base name which is used to compile the filenames for the run files."
583                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
584                               "|Name[string]:path and base file name");
585
586     fNumSubAndFitsData.numSubscriptions = 0;
587     fNumSubAndFitsData.numOpenFits = 0;
588     fNumSubAndFits = new DimDescribedService(GetName() + "/NUM_SUBS", "I:2", fNumSubAndFitsData,
589                               "Shows number of services to which the data logger is currently subscribed and the total number of open files."
590                               "|Subscriptions[int]:number of dim services to which the data logger is currently subscribed."
591                               "|NumOpenFiles[int]:number of files currently open by the data logger");
592
593     //black/white list
594     fHasBlackList = false;
595     fHasWhiteList = false;
596     fBlackList.clear();
597     fWhiteList.clear();
598     //services parameters
599     fDebugIsOn = false;
600     fStatsPeriodDuration = 1.0f;
601     fOpenedFilesIsOn = true;
602     fNumSubAndFitsIsOn = true;
603
604     //provide services control commands
605     AddEvent(fDebugOnOff, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
606             (boost::bind(&DataLogger::SetDebugOnOff, this, _1))
607             ("Switch debug mode on off. Debug mode prints ifnormation about every service written to a file."
608              "|Enable[bool]:Enable of disable debuig mode (yes/no).");
609
610     AddEvent(fStatsPeriod, "F", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
611             (boost::bind(&DataLogger::SetStatsPeriod, this, _1))
612             ("Interval in which the data-logger statitistics service (STATS) is updated."
613              "Interval[s]:Floating point value in seconds.");
614
615     AddEvent(fStartStopOpenedFiles, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
616              (boost::bind(&DataLogger::SetOpenedFilesOnOff ,this, _1))
617              ("Can be used to switch the service off which distributes information about the open files.");
618
619     AddEvent(fStartStopNumSubsAndFits, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
620             (boost::bind(&DataLogger::SetNumSubsAndFitsOnOff, this, _1))
621             ("Can be used to switch the service off which distributes information about the number of subscriptions and open files.");
622
623     fDestructing = false;
624     if(fDebugIsOn)
625     {
626         Debug("DataLogger Init Done.");
627     }
628}
629// --------------------------------------------------------------------------
630//
631//! Checks for changes in the existing services.
632//! Any new service will be added to the service list, while the ones which disappeared are removed.
633//
634//FIXME The service must be udpated so that I get the first notification. This should not be
635bool DataLogger::CheckForServicesUpdate()
636{ 
637    bool serviceUpdated = false;
638    //get the current server list
639    const vector<string> serverList = fServiceList.GetServerList();
640    //first let's remove the servers that may have disapeared
641    //can't treat the erase on maps the same way as for vectors. Do it the safe way instead
642    vector<string> toBeDeleted;
643    for (SubscriptionsListType::iterator cListe = fServiceSubscriptions.begin(); cListe != fServiceSubscriptions.end(); cListe++)
644    {
645        vector<string>::const_iterator givenServers;
646        for (givenServers=serverList.begin(); givenServers!= serverList.end(); givenServers++)
647            if (cListe->first == *givenServers)
648                break;
649        if (givenServers == serverList.end())//server vanished. Remove it
650        {   
651            toBeDeleted.push_back(cListe->first);
652            serviceUpdated = true;
653        }
654    } 
655    for (vector<string>::const_iterator it = toBeDeleted.begin(); it != toBeDeleted.end(); it++)
656        fServiceSubscriptions.erase(*it);
657    //now crawl through the list of servers, and see if there was some updates
658    for (vector<string>::const_iterator i=serverList.begin(); i!=serverList.end();i++)
659    {
660        //skip the two de-fact excluded services
661        //Dim crashes if the publisher subscribes to its own service. This sounds weird, I agree.
662        if ((i->find("DIS_DNS") != string::npos) ||
663            (i->find("DATA_LOGGER") != string::npos))
664            continue;
665        //find the current server in our subscription list   
666        SubscriptionsListType::iterator cSubs = fServiceSubscriptions.find(*i);
667        //get the service list of the current server
668        vector<string> cServicesList = fServiceList.GetServiceList(*i);
669        if (cSubs != fServiceSubscriptions.end())//if the current server already is in our subscriptions
670        {                                         //then check and update our list of subscriptions
671            //first, remove the services that may have dissapeared.
672            map<string, SubscriptionType>::iterator serverSubs;
673            vector<string>::const_iterator givenSubs;
674            toBeDeleted.clear();
675            for (serverSubs=cSubs->second.begin(); serverSubs != cSubs->second.end(); serverSubs++)
676            {
677                for (givenSubs = cServicesList.begin(); givenSubs != cServicesList.end(); givenSubs++)
678                    if (serverSubs->first == *givenSubs)
679                        break;
680                if (givenSubs == cServicesList.end())
681                {
682                    toBeDeleted.push_back(serverSubs->first);
683                    serviceUpdated = true;
684                }   
685            }
686            for (vector<string>::const_iterator it = toBeDeleted.begin(); it != toBeDeleted.end(); it++)
687                cSubs->second.erase(*it);
688            //now check for new services
689            for (givenSubs = cServicesList.begin(); givenSubs != cServicesList.end(); givenSubs++)
690            {
691                if (*givenSubs == "SERVICE_LIST")
692                    continue;
693
694                if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) &&
695                                     (fWhiteList.find(*i + "/" + *givenSubs) == fWhiteList.end()) &&
696                                     (fWhiteList.find("/" + *givenSubs) == fWhiteList.end()))
697                    continue;
698                if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) ||
699                                      (fBlackList.find(*i + "/" + *givenSubs) != fBlackList.end()) ||
700                                      (fBlackList.find("/" + *givenSubs) != fBlackList.end())))
701                    continue;
702
703                if (cSubs->second.find(*givenSubs) == cSubs->second.end())
704                {//service not found. Add it
705                    cSubs->second[*givenSubs].dimInfo = new DimStampedInfo(((*i) + "/" + *givenSubs).c_str(), const_cast<char*>(""), this);
706                    serviceUpdated = true;
707                    if(fDebugIsOn)
708                    {
709                        stringstream str;
710                        str << "Subscribing to service " << *i << "/" << *givenSubs;
711                        Debug(str.str());   
712                    }
713                }   
714            }
715        }
716        else //server not found in our list. Create its entry
717        {
718            fServiceSubscriptions[*i] = map<string, SubscriptionType>();
719            map<string, SubscriptionType>& liste = fServiceSubscriptions[*i];
720            for (vector<string>::const_iterator j = cServicesList.begin(); j!= cServicesList.end(); j++)
721            {
722                if (*j == "SERVICE_LIST")
723                    continue;
724                if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) &&
725                                     (fWhiteList.find(*i + "/" + *j) == fWhiteList.end()) &&
726                                     (fWhiteList.find("/" + *j) == fWhiteList.end()))
727                    continue;
728                if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) ||
729                                      (fBlackList.find(*i + "/" + *j) != fBlackList.end()) ||
730                                      (fBlackList.find("/" + *j) != fBlackList.end())))
731                    continue;
732
733                liste[*j].dimInfo = new DimStampedInfo(((*i) + "/" + (*j)).c_str(), const_cast<char*>(""), this);
734                serviceUpdated = true;
735                if(fDebugIsOn)
736                {
737                    stringstream str;
738                    str << "Subscribing to service " << *i << "/" << *j;
739                    Debug(str.str());   
740                }
741            }
742        }   
743    }
744    return serviceUpdated;
745}
746// --------------------------------------------------------------------------
747//
748//! Destructor
749//
750DataLogger::~DataLogger()
751{
752    if (fDebugIsOn)
753    {
754        Debug("DataLogger destruction starts");   
755    }
756    fDestructing = true;
757    //first let's go to the ready state
758    GoToReadyPlease(); 
759    //release the services subscriptions
760    fServiceSubscriptions.clear();
761    //exit the monitoring loop
762    fContinueMonitoring = false;
763
764    fMonitoringThread.join();
765    //close the files
766    if (fNightlyLogFile.is_open())
767        fNightlyLogFile.close();
768    if (fNightlyReportFile.is_open())
769        fNightlyReportFile.close();
770    if (fRunLogFile.is_open())
771        fRunLogFile.close();
772    if (fRunReportFile.is_open())
773        fRunReportFile.close();
774    delete fOpenedNightlyFiles;
775    delete fOpenedRunFiles;
776    delete fNumSubAndFits;
777#ifdef HAVE_FITS
778    if (fRunFitsFile != NULL)
779        delete fRunFitsFile;
780    fRunFitsFile = NULL;
781#endif
782    if (fDebugIsOn)
783    {
784        Debug("DataLogger desctruction ends");   
785    }
786}
787
788// --------------------------------------------------------------------------
789//
790//! Inherited from DimInfo. Handles all the Infos to which we subscribed, and log them
791//
792void DataLogger::infoHandler()
793{
794    // Make sure getTimestamp is called _before_ getTimestampMillisecs
795    if (fDestructing)
796        return;
797
798    DimInfo* I = getInfo();
799    SubscriptionsListType::iterator x;
800    map<string, SubscriptionType>::iterator y;
801    if (I==NULL)
802    {
803        if (CheckForServicesUpdate())
804        {
805            //services were updated. Notify
806            fNumSubAndFitsData.numSubscriptions = 0;
807            for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
808                fNumSubAndFitsData.numSubscriptions += x->second.size();
809            if (fNumSubAndFitsIsOn)
810            {
811                if (fDebugIsOn)
812                {
813                    stringstream str;
814                    str << "Updating number of subscriptions service: Num Subs=" << fNumSubAndFitsData.numSubscriptions << " Num open FITS=" << fNumSubAndFitsData.numOpenFits;
815                    Debug(str.str());   
816                }
817                fNumSubAndFits->updateService();
818            }
819        }
820        return;
821    }
822    //check if the service pointer corresponds to something that we subscribed to
823    //this is a fix for a bug that provides bad Infos when a server starts
824    bool found = false;
825    for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
826    {//find current service is subscriptions
827        for (y=x->second.begin(); y!=x->second.end();y++)
828            if (y->second.dimInfo == I)
829            {
830                found = true;   
831                break;
832            }
833        if (found)
834            break;
835    }
836    if (!found)
837        return;
838    if (I->getSize() <= 0)
839        return;
840
841        // Make sure that getTimestampMillisecs is NEVER called before
842        // getTimestamp is properly called
843        // check that the message has been updated by something, i.e. must be different from its initial value
844    if (I->getTimestamp() == 0)
845        return;
846
847    CheckForRunNumber(I);
848
849    if (fPreviousRunNumber != fRunNumber)
850    {//run number has changed. close and reopen run files.
851        StopRunPlease();
852        StartRunPlease();
853        fPreviousRunNumber = fRunNumber;
854    }
855
856    ReportPlease(I, y->second);
857
858}
859
860// --------------------------------------------------------------------------
861//
862//! Checks whether or not the current info is a run number.
863//! If so, then remember it. A run number is required to open the run-log file
864//! @param I
865//!        the current DimInfo
866//
867void DataLogger::CheckForRunNumber(DimInfo* I)
868{
869    if (strstr(I->getName(), fRunNumberInfo) != NULL)
870    {//assumes that the run number is an integer
871        fRunNumber = I->getInt();   
872        stringstream str;
873        str << "New run number is " << fRunNumber;
874        Message(str.str());
875    }
876}
877
878// --------------------------------------------------------------------------
879//
880//! write infos to log files.
881//! @param I
882//!     The current DimInfo
883//! @param sub
884//!        The dataLogger's subscription corresponding to this DimInfo
885//
886void DataLogger::ReportPlease(DimInfo* I, SubscriptionType& sub)
887{
888    //should we log or report this info ? (i.e. is it a message ?)
889    bool isItaReport = ((strstr(I->getName(), "Message") == NULL) && (strstr(I->getName(), "MESSAGE") == NULL));
890    if (I->getFormat()[0] == 'C')
891        isItaReport = false;
892   
893    if (!fNightlyReportFile.is_open())
894        return;
895
896    //create the converter for that service
897    if (sub.fConv == NULL && isItaReport)
898    {
899        //trick the converter in case of 'C'. why do I do this ? well simple: the converter checks that the right number
900        //of bytes was written. because I skip 'C' with fits, the bytes will not be allocated, hence the "size copied ckeck"
901        //of the converter will fail, hence throwing an exception.
902        string fakeFormat(I->getFormat());
903        if (fakeFormat[fakeFormat.size()-1] == 'C')
904            fakeFormat = fakeFormat.substr(0, fakeFormat.size()-1);
905        sub.fConv = new Converter(Out(), I->getFormat());   
906        if (!sub.fConv)
907        {
908            stringstream str;
909            str << "Couldn't properly parse the format... service " << sub.dimInfo->getName() << " ignored.";
910            Error(str);
911            return;   
912        }
913    }
914       
915    //construct the header
916    stringstream header;
917    Time cTime(I->getTimestamp(), I->getTimestampMillisecs()*1000);
918    fQuality = I->getQuality();
919    fMjD = cTime.Mjd();
920
921    if (isItaReport)
922    {
923        //write text header
924        header << I->getName() << " " << fQuality << " ";
925        header << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
926        header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
927        header << cTime.ms() << " " << I->getTimestamp() << " ";
928
929        string text;
930        try
931        {
932            text = sub.fConv->GetString(I->getData(), I->getSize());
933        }
934        catch (const runtime_error &e)
935        {
936            Out() << kRed << e.what() << endl;
937            stringstream str;
938            str << "Could not properly parse the data for service " << sub.dimInfo->getName();
939            str << " reason: " << e.what() << ". Entry ignored";
940            Error(str);
941            return;
942        }
943
944        if (text.empty())
945        {
946            stringstream str;
947            str << "Service " << sub.dimInfo->getName() << " sent an empty string";
948            Info(str);
949            return;
950        }
951        //replace bizarre characters by white space
952        replace(text.begin(), text.end(), '\n', '\\');
953        replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
954       
955        //write entry to Nightly report
956        if (fNightlyReportFile.is_open())
957        {
958            if (fDebugIsOn)
959            {
960                stringstream str;
961                str << "Writing: \"" << header.str() << text << "\" to Nightly report file";
962                Debug(str.str());   
963            }
964            fNightlyReportFile << header.str() << text << endl;
965            //check if either eof, bailbit or batbit are set
966            if (!fNightlyReportFile.good())
967            {
968                Error("An error occured while writing to the nightly report file. Closing it");
969                if (fNightlyReportFile.is_open())
970                    fNightlyReportFile.close();
971            }
972        }
973        //write entry to run-report
974        if (fRunReportFile.is_open())
975        {
976            if (fDebugIsOn)
977            {
978                stringstream str;
979                str << "Writing: \"" << header.str() << text << "\" to Run report file";
980                Debug(str.str());   
981            }
982            fRunReportFile << header.str() << text << endl;
983            if (!fRunReportFile.good())
984            {
985                Error("An error occured while writing to the run report file. Closing it.");
986                if (fRunReportFile.is_open())
987                    fRunReportFile.close();   
988            }
989        }
990    }
991    else
992    {//write entry to both Nightly and run logs
993        string n = I->getName();
994        stringstream msg;
995        msg << n << ": " << I->getString();//n.substr(0, n.find_first_of('/')) << ": " << I->getString();
996
997        if (fNightlyLogFile.is_open())
998        {
999            if (fDebugIsOn)
1000            {
1001                stringstream str;
1002                str << "Writing: \"" << msg.str() << "\" to Nightly log file";
1003                Debug(str.str());   
1004            }
1005            MessageImp nightlyMess(fNightlyLogFile);
1006            nightlyMess.Write(cTime, msg.str().c_str(), fQuality);
1007            if (!fNightlyLogFile.good())
1008            {
1009                Error("An error occured while writing to the nightly log file. Closing it.");
1010                if (fNightlyLogFile.is_open())
1011                    fNightlyLogFile.close();   
1012            }
1013        }
1014        if (fRunLogFile.is_open())
1015        {
1016            if (fDebugIsOn)
1017            {
1018                stringstream str;
1019                str << "Writing: \"" << msg.str() << "\" to Run log file";
1020                Debug(str.str());   
1021            }
1022            MessageImp runMess(fRunLogFile);
1023            runMess.Write(cTime, msg.str().c_str(), fQuality);
1024            if (!fRunLogFile.good())
1025            {
1026                Error("An error occured while writing to the run log file. Closing it.");
1027                if (fRunLogFile.is_open())
1028                    fRunLogFile.close();   
1029            }
1030        }
1031    }
1032
1033#ifdef HAVE_FITS
1034    if (isItaReport)
1035    {
1036        if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen())
1037            OpenFITSFilesPlease(sub);   
1038        WriteToFITS(sub);
1039    }   
1040#endif
1041
1042}
1043
1044// --------------------------------------------------------------------------
1045//
1046//! write messages to logs.
1047//! @param evt
1048//!        the current event to log
1049//! @returns
1050//!        the new state. Currently, always the current state
1051//!
1052//! @deprecated
1053//!    I guess that this function should not be any longer
1054//
1055//Otherwise re-write it properly with the MessageImp class
1056int DataLogger::LogMessagePlease(const Event& evt)
1057{
1058    if (!fNightlyLogFile.is_open())
1059        return GetCurrentState();
1060    Warn("LogMessagePlease has not been checked nor updated since a long while. Undefined behavior to be expected");
1061    stringstream header;
1062    const Time& cTime = evt.GetTime();
1063    header << evt.GetName() << " " << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
1064    header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
1065    header << cTime.ms() << " ";
1066       
1067    const Converter conv(Out(), evt.GetFormat());
1068    if (!conv)
1069    {
1070        Error("Couldn't properly parse the format... ignored.");
1071        return GetCurrentState();
1072    }
1073
1074    string text;
1075    try
1076    {
1077        text = conv.GetString(evt.GetData(), evt.GetSize());
1078    }
1079    catch (const runtime_error &e)
1080    {
1081        Out() << kRed << e.what() << endl;
1082        Error("Couldn't properly parse the data... ignored.");
1083        return GetCurrentState();
1084    }
1085
1086    if (text.empty())
1087        return GetCurrentState();
1088
1089    //replace bizarre characters by white space
1090    replace(text.begin(), text.end(), '\n', '\\');
1091    replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
1092    if (fDebugIsOn)
1093    {
1094        stringstream str;
1095        str << "Logging: \"" << header << text << "\"";
1096        Debug(str.str());   
1097    }
1098   
1099    if (fNightlyLogFile.is_open())
1100    {
1101        fNightlyLogFile << header;
1102        if (!fNightlyLogFile.good())
1103        {
1104            Error("An error occured while writing to the run log file. Closing it.");
1105            if (fNightlyLogFile.is_open())
1106                fNightlyLogFile.close();   
1107        }
1108    }
1109    if (fRunLogFile.is_open())
1110    {
1111        fRunLogFile << header;
1112        if (!fRunLogFile.good())
1113        {
1114            Error("An error occured while writing to the run log file. Closing it.");
1115            if (fRunLogFile.is_open())
1116                fRunLogFile.close();   
1117        }
1118    }
1119    if (fNightlyLogFile.is_open())
1120    {
1121        fNightlyLogFile << text;
1122        if (!fNightlyLogFile.good())
1123        {
1124            Error("An error occured while writing to the run log file. Closing it.");
1125            if (fNightlyLogFile.is_open())
1126                fNightlyLogFile.close();   
1127        }
1128    }
1129    if (fRunLogFile.is_open())
1130    {
1131        fRunLogFile << text;
1132        if (!fRunLogFile.good())
1133        {
1134            Error("An error occured while writing to the run log file. Closing it.");
1135            if (fRunLogFile.is_open())
1136                fRunLogFile.close();   
1137        }
1138    }
1139    return GetCurrentState();
1140}
1141// --------------------------------------------------------------------------
1142//
1143//! print the dataLogger's current state. invoked by the PRINT command
1144//! @param evt
1145//!        the current event. Not used by the method
1146//! @returns
1147//!        the new state. Which, in that case, is the current state
1148//!
1149int DataLogger::PrintStatePlease(const Event& )
1150{
1151    Message("-----------------------------------------");
1152    Message("------ DATA LOGGER CURRENT STATE --------");
1153    Message("-----------------------------------------");
1154    //print the path configuration
1155    string actualTargetDir;
1156    if (fNightlyFileName == ".")
1157    {
1158        char currentPath[FILENAME_MAX];
1159        if (getcwd(currentPath, sizeof(currentPath)))
1160            actualTargetDir = currentPath;
1161    }
1162    else
1163        actualTargetDir = fNightlyFileName;
1164    Message("Nightly Path: " + actualTargetDir);
1165    if (fRunFileName == ".")
1166    {
1167        char currentPath[FILENAME_MAX];
1168        if (getcwd(currentPath, sizeof(currentPath)))
1169            actualTargetDir = currentPath;
1170    }
1171    else
1172        actualTargetDir = fRunFileName;
1173    Message("Run Path: " + actualTargetDir);
1174    stringstream str;
1175    str << "Run Number: " << fRunNumber;
1176    Message(str.str());
1177    Message("----------- OPENED FILES ----------------");
1178    //print all the open files.
1179    if (fNightlyLogFile.is_open())
1180        Message("Nightly Log..........OPEN");
1181    else
1182        Message("Nightly log........CLOSED");
1183    if (fNightlyReportFile.is_open())
1184        Message("Nightly Report.......OPEN");
1185    else
1186        Message("Nightly Report.....CLOSED");
1187    if (fRunLogFile.is_open())
1188        Message("Run Log..............OPEN");
1189    else
1190        Message("Run Log............CLOSED");
1191    if (fRunReportFile.is_open())
1192        Message("Run Report...........OPEN");
1193    else
1194        Message("Run Report.........CLOSED");
1195    bool statWarning = false;
1196    DataLoggerStats statVar;
1197    calculateTotalSizeWritten(statVar, statWarning, false);
1198    Message("---------------- STATS ------------------");
1199    str.str("");
1200    str << "Total Size written: " << statVar.sizeWritten << " bytes.";
1201        Message(str.str());
1202    str.str("");
1203    str << "Disk free space:    " << statVar.freeSpace   << " bytes.";
1204        Message(str.str());
1205    str.str("");
1206    str << "Statistics are updated every " << fStatsPeriodDuration << " seconds";
1207    if (fStatsPeriodDuration != 0)
1208        Message(str);
1209    else
1210        Message("Statistics updates are currently disabled");
1211    Message("----------- DIM SUBSCRIPTIONS -----------");
1212
1213    str.str("");
1214    str << "There are " << fNumSubAndFitsData.numSubscriptions << " active DIM subscriptions:";
1215    Message(str.str());
1216
1217    for (map<const string, map<string, SubscriptionType> >::const_iterator it=fServiceSubscriptions.begin(); it!= fServiceSubscriptions.end();it++)
1218    {
1219        Message("Server "+it->first);
1220        for (map<string, SubscriptionType>::const_iterator it2=it->second.begin(); it2!=it->second.end(); it2++)
1221            Message(" -> "+it2->first);
1222    }
1223    if (fHasBlackList)
1224    {
1225        Message("------------- BLOCK LIST ----------------");
1226        for (set<string>::iterator it=fBlackList.begin(); it != fBlackList.end(); it++)
1227            Message(*it);
1228    }
1229    if (fHasWhiteList)
1230    {
1231            Message("----------- ALLOW LIST ------------------");
1232            for (set<string>::iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++)
1233                Message(*it);
1234    }
1235    if (fGrouping.size() != 0)
1236    {
1237            Message("--------- GROUPING LIST -----------------");
1238            Message("The following servers and/or services will be grouping under a single run fits file:");
1239            for (set<string>::iterator it=fGrouping.begin(); it != fGrouping.end(); it++)
1240                Message(*it);
1241    }
1242    Message("-----------------------------------------");
1243    Message("------ END OF DATA LOGGER STATE ---------");
1244    Message("-----------------------------------------");
1245
1246    return GetCurrentState();
1247}
1248
1249// --------------------------------------------------------------------------
1250//
1251//! turn debug mode on and off
1252//! @param evt
1253//!        the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1
1254//! @returns
1255//!        the new state. Which, in that case, is the current state
1256//!
1257int DataLogger::SetDebugOnOff(const Event& evt)
1258{
1259    bool backupDebug = fDebugIsOn;
1260    fDebugIsOn = evt.GetBool();
1261    if (fDebugIsOn == backupDebug)
1262        Warn("Warning: debug mode was already in the requested state");
1263    else
1264    {
1265        stringstream str;
1266        str << "Debug mode is now " << fDebugIsOn;
1267        Message(str.str());
1268    }
1269    return GetCurrentState();
1270}
1271// --------------------------------------------------------------------------
1272//
1273//! set the statistics update period duration. 0 disables the statistics
1274//! @param evt
1275//!        the current event. contains the new duration.
1276//! @returns
1277//!        the new state. Which, in that case, is the current state
1278//!
1279int DataLogger::SetStatsPeriod(const Event& evt)
1280{
1281    float backupDuration = fStatsPeriodDuration;
1282    fStatsPeriodDuration = evt.GetFloat();
1283    if (fStatsPeriodDuration < 0)
1284    {
1285        Error("Statistics period duration should be greater than zero. Discarding provided value.");
1286        fStatsPeriodDuration = backupDuration;
1287        return GetCurrentState();   
1288    }
1289    if (fStatsPeriodDuration != fStatsPeriodDuration)
1290    {
1291        Error("Provided duration does not appear to be a valid float. discarding it.");
1292        fStatsPeriodDuration = backupDuration;
1293        return GetCurrentState();   
1294    }
1295    if (backupDuration == fStatsPeriodDuration)
1296        Warn("Warning: statistics period was not modified: supplied value already in use");
1297    else
1298    {
1299        if (fStatsPeriodDuration == 0.0f)
1300            Message("Statistics are now OFF");
1301        else
1302        {
1303            stringstream str;
1304            str << "Statistics period is now " << fStatsPeriodDuration << " seconds";
1305            Message(str.str());     
1306        }   
1307    }
1308    return GetCurrentState();
1309}
1310// --------------------------------------------------------------------------
1311//
1312//! set the opened files service on or off.
1313//! @param evt
1314//!        the current event. contains the instruction string. similar to setdebugonoff
1315//! @returns
1316//!        the new state. Which, in that case, is the current state
1317//!
1318int DataLogger::SetOpenedFilesOnOff(const Event& evt)
1319{
1320    bool backupOpened = fOpenedFilesIsOn;
1321    fOpenedFilesIsOn = evt.GetBool();
1322    if (fOpenedFilesIsOn == backupOpened)
1323        Warn("Warning: opened files service mode was already in the requested state");
1324    else
1325    {
1326        stringstream str;
1327        str << "Opened files service mode is now " << fOpenedFilesIsOn;
1328        Message(str.str());
1329    }
1330    return GetCurrentState();
1331   
1332}
1333// --------------------------------------------------------------------------
1334//
1335//! set the number of subscriptions and opened fits on and off
1336//! @param evt
1337//!        the current event. contains the instruction string. similar to setdebugonoff
1338//! @returns
1339//!        the new state. Which, in that case, is the current state
1340//!
1341int DataLogger::SetNumSubsAndFitsOnOff(const Event& evt)
1342{
1343    bool backupSubs = fNumSubAndFitsIsOn;
1344    fNumSubAndFitsIsOn = evt.GetBool();
1345    if (fNumSubAndFitsIsOn == backupSubs)
1346        Warn("Warning: Number of subscriptions service mode was already in the requested state");
1347    else
1348    {
1349        stringstream str;
1350        str << "Number of subscriptions service mode is now " << fNumSubAndFitsIsOn;
1351        Message(str.str());
1352    }
1353    return GetCurrentState();
1354}
1355// --------------------------------------------------------------------------
1356//
1357//!    Sets the path to use for the Nightly log file.
1358//! @param evt
1359//!     the event transporting the path
1360//! @returns
1361//!        currently only the current state.
1362//
1363int DataLogger::ConfigureNightlyFileName(const Event& evt)
1364{
1365    if (evt.GetText() != NULL)
1366    {
1367        fNightlyFileName = string(evt.GetText());
1368        Message("New Nightly folder specified: " + fNightlyFileName);
1369    }
1370    else
1371        Error("Empty Nightly folder given. Please specify a valid path.");
1372
1373    return GetCurrentState();
1374}
1375// --------------------------------------------------------------------------
1376//
1377//! Sets the path to use for the run log file.
1378//! @param evt
1379//!        the event transporting the path
1380//! @returns
1381//!     currently only the current state
1382int DataLogger::ConfigureRunFileName(const Event& evt)
1383{
1384    if (evt.GetText() != NULL)
1385    {
1386        fRunFileName = string(evt.GetText());
1387        Message("New Run folder specified: " + fRunFileName);
1388    }
1389    else
1390        Error("Empty Nightly folder given. Please specify a valid path");
1391
1392    return GetCurrentState();
1393}
1394// --------------------------------------------------------------------------
1395//
1396//! Sets the run number.
1397//! @param evt
1398//!        the event transporting the run number
1399//! @returns
1400//!     currently only the current state
1401int DataLogger::ConfigureRunNumber(const Event& evt)
1402{
1403    fRunNumber = evt.GetInt();
1404    stringstream str;
1405    str << "The new run number is: " << fRunNumber;
1406    Message(str.str());
1407    return GetCurrentState();
1408}
1409// --------------------------------------------------------------------------
1410//
1411//! Notifies the DIM service that a particular file was opened
1412//! @ param name the base name of the opened file, i.e. without path nor extension.
1413//!     WARNING: use string instead of string& because I pass values that do not convert to string&.
1414//!        this is not a problem though because file are not opened so often.
1415//! @ param type the type of the opened file. 0 = none open, 1 = log, 2 = text, 4 = fits
1416inline void DataLogger::NotifyOpenedFile(string name, int type, DimDescribedService* service)
1417{
1418    if (fOpenedFilesIsOn)
1419    {
1420        if (fDebugIsOn)
1421        {
1422            stringstream str;
1423            str << "Updating files service " << service->getName() << "with code: " << type << " and file: " << name;
1424            Debug(str.str());
1425            str.str("");
1426            str << "Num subs: " << fNumSubAndFitsData.numSubscriptions << " Num open FITS: " << fNumSubAndFitsData.numOpenFits;
1427            Debug(str.str());
1428        }
1429        OpenFileToDim fToDim;
1430        fToDim.code = type;
1431        memcpy(fToDim.fileName, name.c_str(), name.size()+1);
1432        service->setData(reinterpret_cast<void*>(&fToDim), name.size()+1+sizeof(int));
1433        service->setQuality(0);
1434        service->updateService();
1435    }
1436}
1437// --------------------------------------------------------------------------
1438//
1439//! Implements the Start transition.
1440//! Concatenates the given path for the Nightly file and the filename itself (based on the day),
1441//! and tries to open it.
1442//! @returns
1443//!        kSM_NightlyOpen if success, kSM_BadNightlyConfig if failure
1444int DataLogger::StartPlease()
1445{
1446    if (fDebugIsOn)
1447    {
1448        Debug("Starting...");   
1449    }
1450    Time time;
1451    stringstream sTime;
1452    sTime << time.Y() << "_" << time.M() << "_" << time.D();
1453
1454    fFullNightlyLogFileName = fNightlyFileName + '/' + sTime.str() + ".log"; 
1455    fNightlyLogFile.open(fFullNightlyLogFileName.c_str(), ios_base::out | ios_base::app);
1456    if (errno != 0)
1457    {
1458        stringstream str;
1459        str << "Unable to open Nightly Log " << fFullNightlyLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1460        Error(str);   
1461    }
1462    fFullNightlyReportFileName = fNightlyFileName + '/' + sTime.str() + ".rep";
1463    fNightlyReportFile.open(fFullNightlyReportFileName.c_str(), ios_base::out | ios_base::app);
1464    if (errno != 0)
1465    {
1466        stringstream str;
1467        str << "Unable to open Nightly Report " << fFullNightlyReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1468        Error(str);   
1469    }
1470
1471    if (!fNightlyLogFile.is_open() || !fNightlyReportFile.is_open())
1472    {   
1473        stringstream str;
1474        str << "Something went wrong while openning nightly files " << fFullNightlyLogFileName << " and " << fFullNightlyReportFileName;
1475        Error(str.str());
1476        return kSM_BadNightlyConfig;
1477    }
1478    //get the size of the newly opened file.
1479    struct stat st;
1480    stat(fFullNightlyLogFileName.c_str(), &st);
1481    fBaseSizeNightly = st.st_size;   
1482    stat(fFullNightlyReportFileName.c_str(), &st);
1483    fBaseSizeNightly += st.st_size;
1484    fFileSizesMap.clear();
1485    fBaseSizeRun = 0;
1486    fPreviousSize = 0;
1487    //notify that files were opened
1488    string actualTargetDir;
1489    if (fNightlyFileName == ".")
1490    {
1491        char currentPath[FILENAME_MAX];
1492        if (!getcwd(currentPath, sizeof(currentPath)))
1493        {
1494            if (errno != 0)
1495            {
1496                stringstream str;
1497                str << "Unable retrieve current path" << ". Reason: " << strerror(errno) << " [" << errno << "]";
1498                Error(str);
1499            }
1500        }
1501        actualTargetDir = currentPath;
1502    }
1503    else
1504    {
1505        actualTargetDir = fNightlyFileName;   
1506    }
1507    //notify that a new file has been opened.
1508    NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 3, fOpenedNightlyFiles);
1509
1510    fOpenedNightlyFits.clear();
1511   
1512    return kSM_NightlyOpen;     
1513}
1514
1515#ifdef HAVE_FITS
1516// --------------------------------------------------------------------------
1517//
1518//! open if required a the FITS files corresponding to a given subscription
1519//! @param sub
1520//!     the current DimInfo subscription being examined
1521void DataLogger::OpenFITSFilesPlease(SubscriptionType& sub)
1522{
1523    string serviceName(sub.dimInfo->getName());
1524    //we must check if we should group this service subscription to a single fits file before we replace the / by _
1525    bool hasGrouping = false;
1526    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
1527    {//will we find this service in the grouping list ?
1528        for (set<string>::iterator it=fGrouping.begin(); it!=fGrouping.end(); it++)
1529        {
1530            if (serviceName.find(*it) != string::npos)
1531            {
1532                hasGrouping = true;
1533                break;
1534            }
1535        }
1536    }
1537    for (unsigned int i=0;i<serviceName.size(); i++)
1538    {
1539        if (serviceName[i] == '/')
1540        {
1541            serviceName[i] = '_';
1542            break;   
1543        }   
1544    }
1545    Time time;
1546    stringstream sTime;
1547    sTime << time.Y() << "_" << time.M() << "_" << time.D();
1548    //we open the NightlyFile anyway, otherwise this function shouldn't have been called.
1549    if (!sub.nightlyFile.IsOpen())
1550    {
1551        string fileNameOnly = sTime.str() + '_' + serviceName + ".fits";
1552        string partialName = fNightlyFileName + '/' + fileNameOnly;
1553        AllocateFITSBuffers(sub);
1554        //get the size of the file we're about to open
1555        if (fFileSizesMap.find(partialName) == fFileSizesMap.end())
1556        {
1557            struct stat st;
1558            if (!stat(partialName.c_str(), &st))
1559                fBaseSizeNightly += st.st_size;
1560            fFileSizesMap[partialName] = 0;
1561            //remember that this file was opened.
1562            fOpenedNightlyFits[fileNameOnly].push_back(serviceName);
1563        }
1564        sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out());
1565
1566        //notify the opening
1567        string actualTargetDir;
1568        if (fNightlyFileName == ".")
1569        {
1570            char currentPath[FILENAME_MAX];
1571            if (getcwd(currentPath, sizeof(currentPath)))
1572                actualTargetDir = currentPath;
1573        }
1574        else
1575        {
1576            actualTargetDir = fNightlyFileName;   
1577        }       
1578        NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 7, fOpenedNightlyFiles);
1579        if (fNumSubAndFitsIsOn)
1580            fNumSubAndFits->updateService();
1581        if (fDebugIsOn)
1582        {
1583            stringstream str;
1584            str << "Opened Nightly FITS: " << partialName << " and table: FACT-" << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
1585            Debug(str.str());   
1586        }
1587    }
1588    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
1589    {//buffer for the run file have already been allocated when doing the Nightly file
1590        stringstream sRun;
1591        sRun << fRunNumber;
1592        string fileNameOnly;
1593        string partialName;
1594
1595        if (hasGrouping)
1596        {
1597            fileNameOnly = sRun.str() + "_group.fits";
1598            partialName = fRunFileName + '/' + fileNameOnly;
1599        }
1600        else
1601        {
1602            fileNameOnly = sRun.str() + '_' + serviceName + ".fits";
1603            partialName = fRunFileName + '/' + fileNameOnly;
1604        }
1605        //get the size of the file we're about to open
1606        if (fFileSizesMap.find(partialName) == fFileSizesMap.end())
1607        {
1608            struct stat st;
1609            if (!stat(partialName.c_str(), &st))
1610                fBaseSizeRun += st.st_size;
1611            else
1612                fBaseSizeRun = 0;
1613            fFileSizesMap[partialName] = 0;
1614            fOpenedRunFits[fileNameOnly].push_back(serviceName);
1615        }
1616        else
1617            if (hasGrouping)
1618            {//most likely I should add this service name.
1619             //the only case for which I should not add it is if a service disapeared, hence the file was closed
1620             //and reopened again. Unlikely to happen, but well it may
1621                bool found = false;
1622                for (vector<string>::iterator it=fOpenedRunFits[fileNameOnly].begin(); it!=fOpenedRunFits[fileNameOnly].end(); it++)
1623                    if (*it == serviceName)
1624                    {
1625                        found = true;
1626                        break;
1627                    }
1628                if (!found)
1629                    fOpenedRunFits[fileNameOnly].push_back(serviceName);
1630            }
1631        if (hasGrouping && fRunFitsFile == NULL)
1632            try
1633            {
1634                fRunFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write);
1635                (fNumSubAndFitsData.numOpenFits)++;
1636            }   
1637            catch (CCfits::FitsError e)
1638            {
1639                stringstream str;
1640                str << "Could not open FITS Run file " << partialName << " reason: " << e.message();
1641                Error(str);
1642                fRunFitsFile = NULL;
1643            }
1644
1645        string actualTargetDir;
1646        if (fRunFileName == ".")
1647        {
1648            char currentPath[FILENAME_MAX];
1649            if (getcwd(currentPath, sizeof(currentPath)))
1650                actualTargetDir = currentPath;
1651        }
1652        else
1653        {
1654           actualTargetDir = fRunFileName;
1655        }
1656        NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 7, fOpenedRunFiles);// + '_' + serviceName, 4);
1657        if (hasGrouping)
1658            sub.runFile.Open(partialName, serviceName, fRunFitsFile, &fNumSubAndFitsData.numOpenFits, this);//Out());
1659        else
1660            sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out());
1661
1662       if (fNumSubAndFitsIsOn)
1663           fNumSubAndFits->updateService();
1664           if (fDebugIsOn)
1665        {
1666            stringstream str;
1667            str << "Opened Run FITS: " << partialName << " and table: FACT-" << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
1668            Debug(str.str());   
1669        }
1670    }
1671}   
1672// --------------------------------------------------------------------------
1673//
1674void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
1675{
1676    int size = sub.dimInfo->getSize();
1677     
1678    //Init the time columns of the file
1679    Description dateDesc(string("Time"), string("Modified Julian Date"), string("MjD"));
1680    sub.nightlyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1681    sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1682
1683    Description QoSDesc("Qos", "Quality of service", "None");
1684    sub.nightlyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1685    sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1686
1687    const Converter::FormatList flist = sub.fConv->GetList();
1688    // Compilation failed
1689    if (flist.empty() || flist.back().first.second!=0)
1690    {
1691        Error("Compilation of format string failed.");
1692        return;
1693    }
1694
1695    //we've got a nice structure describing the format of this service's messages.
1696    //Let's create the appropriate FITS columns
1697    vector<string> dataFormatsLocal;
1698    for (unsigned int i=0;i<flist.size()-1;i++)
1699    {
1700         stringstream dataQualifier;
1701
1702         dataQualifier << flist[i].second.first;
1703         switch (flist[i].first.first->name()[0])
1704         {
1705             case 'c':
1706             case 'C':
1707                 dataQualifier.str("S");
1708             break;
1709             case 's':
1710                 dataQualifier << "I";
1711             break;
1712             case 'i':
1713             case 'I':
1714                 dataQualifier << "J";
1715             break;
1716             case 'l':
1717             case 'L':
1718                 dataQualifier << "J";
1719             break;
1720             case 'f':
1721             case 'F':
1722                 dataQualifier << "E";
1723             break;
1724             case 'd':
1725             case 'D':
1726                 dataQualifier << "D";
1727             break;
1728             case 'x':
1729             case 'X':
1730                 dataQualifier << "K";
1731             break;
1732             case 'S':
1733                 //for strings, the number of elements I get is wrong. Correct it
1734                 dataQualifier.str(""); //clear
1735                 dataQualifier << size-1 <<  "A";
1736                 size = size-1;
1737             break;
1738             
1739             default:
1740                 Fatal("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 1198.");
1741         };
1742         //we skip the variable length strings for now (in fits only)
1743         if (dataQualifier.str() != "S")
1744             dataFormatsLocal.push_back(dataQualifier.str());
1745     }
1746     sub.nightlyFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size);
1747     sub.runFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size);
1748}
1749// --------------------------------------------------------------------------
1750//
1751//! write a dimInfo data to its corresponding FITS files
1752//
1753void DataLogger::WriteToFITS(SubscriptionType& sub)
1754{
1755        //nightly File status (open or not) already checked
1756        if (sub.nightlyFile.IsOpen())
1757        {
1758            sub.nightlyFile.Write(sub.fConv);
1759            if (fDebugIsOn)
1760            {
1761                Debug("Writing to nightly FITS " + sub.nightlyFile.fFileName);   
1762            }
1763        }
1764        if (sub.runFile.IsOpen())
1765        {
1766            sub.runFile.Write(sub.fConv);
1767            if (fDebugIsOn)
1768            {
1769                Debug("Writing to Run FITS " + sub.runFile.fFileName);   
1770            }
1771        }
1772}
1773#endif //if has_fits
1774// --------------------------------------------------------------------------
1775//
1776//! Implements the StartRun transition.
1777//! Concatenates the given path for the run file and the filename itself (based on the run number),
1778//! and tries to open it.
1779//! @returns
1780//!        kSM_Logging if success, kSM_BadRunConfig if failure.
1781int DataLogger::StartRunPlease()
1782{
1783    if (fDebugIsOn)
1784    {
1785        Debug("Starting Run Logging...");   
1786    }
1787    //attempt to open run file with current parameters
1788//    if (fRunNumber == -1)
1789//        return kSM_BadRunConfig;
1790    stringstream sRun;
1791    sRun << fRunNumber;
1792    fFullRunLogFileName = fRunFileName + '/' + sRun.str() + ".log";
1793    fRunLogFile.open(fFullRunLogFileName.c_str(), ios_base::out | ios_base::app); //maybe should be app instead of ate
1794    if (errno != 0)
1795    {
1796        stringstream str;
1797        str << "Unable to open run Log " << fFullRunLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1798        Error(str);   
1799    }
1800    fFullRunReportFileName = fRunFileName + '/' + sRun.str() + ".rep";
1801    fRunReportFile.open(fFullRunReportFileName.c_str(), ios_base::out | ios_base::app);
1802    if (errno != 0)
1803    {
1804        stringstream str;
1805        str << "Unable to open run report " << fFullRunReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1806        Error(str);   
1807    }
1808   
1809    if (!fRunLogFile.is_open() || !fRunReportFile.is_open())
1810    {
1811        stringstream str;
1812        str << "Something went wrong while openning nightly files " << fFullRunLogFileName << " and " << fFullRunReportFileName;
1813        Error(str.str());
1814        return kSM_BadRunConfig;
1815    }
1816    //get the size of the newly opened file.
1817    struct stat st;
1818    fBaseSizeRun = 0;
1819    if (fFileSizesMap.find(fFullRunLogFileName) == fFileSizesMap.end())
1820    {
1821        stat(fFullRunLogFileName.c_str(), &st);
1822        if (errno != 0)
1823        {
1824            stringstream str;
1825            str << "Unable to stat " << fFullRunLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1826            Error(str);   
1827        }
1828        else
1829            fBaseSizeRun += st.st_size;
1830        fFileSizesMap[fFullRunLogFileName] = 0;
1831    }
1832    if (fFileSizesMap.find(fFullRunReportFileName) == fFileSizesMap.end())
1833    {
1834        stat(fFullRunReportFileName.c_str(), &st);
1835        if (errno != 0)
1836        {
1837            stringstream str;
1838            str << "Unable to stat " << fFullRunReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1839            Error(str);   
1840        }
1841        else
1842            fBaseSizeRun += st.st_size;
1843        fFileSizesMap[fFullRunReportFileName] = 0;
1844    }
1845    string actualTargetDir;
1846    if (fRunFileName == ".")
1847    {
1848        char currentPath[FILENAME_MAX];
1849        if (!getcwd(currentPath, sizeof(currentPath)))
1850        {
1851            if (errno != 0)
1852            {
1853                stringstream str;
1854                str << "Unable to retrieve the current path" << ". Reason: " << strerror(errno) << " [" << errno << "]";
1855                Error(str);
1856            }
1857        }
1858        actualTargetDir = currentPath;
1859    }
1860    else
1861    {
1862        actualTargetDir = fRunFileName;   
1863    }       
1864    NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 3, fOpenedRunFiles);
1865   
1866    fOpenedRunFits.clear();
1867
1868    return kSM_Logging;
1869}
1870#ifdef HAVE_FITS
1871void DataLogger::CreateFitsGrouping(bool runGroup)
1872{
1873    if (fDebugIsOn)
1874    {
1875        stringstream str;
1876        str << "Creating fits group for ";
1877        if (runGroup)
1878            str << "run files";
1879        else
1880            str << "nightly files";
1881        Debug(str.str());
1882    }
1883    //create the FITS group corresponding to the ending run.
1884    CCfits::FITS* groupFile;
1885    map<string, vector<string> >& filesToGroup = runGroup? fOpenedRunFits : fOpenedNightlyFits;
1886    unsigned int numFilesToGroup = 0;
1887    for (map<string, vector<string> >::iterator it=filesToGroup.begin(); it != filesToGroup.end(); it++)
1888    {
1889        numFilesToGroup += it->second.size();
1890    }
1891    if (fDebugIsOn)
1892    {
1893        stringstream str;
1894        str << "There are " << numFilesToGroup << " tables to group";
1895        Debug(str.str());
1896    }
1897    if (numFilesToGroup <= 1)
1898    {
1899        filesToGroup.clear();
1900        return;
1901    }
1902    stringstream groupName;
1903    if (runGroup)
1904    {
1905        groupName << fRunFileName << '/' << fRunNumber << ".fits";
1906    }
1907    else
1908    {
1909        Time time;
1910        stringstream sTime;
1911        sTime << time.Y() << "_" << time.M() << "_" << time.D();
1912
1913        groupName << fNightlyFileName << '/' << sTime.str() << ".fits";
1914    }
1915    CCfits::Table* groupTable;
1916    int maxCharLength = 50;//FILENAME_MAX;
1917    try
1918    {
1919        groupFile = new CCfits::FITS(groupName.str(), CCfits::RWmode::Write);
1920        //setup the column names
1921        stringstream pathTypeName;
1922        pathTypeName << maxCharLength << "A";
1923        vector<string> names;
1924        vector<string> dataTypes;
1925        names.push_back("MEMBER_XTENSION");
1926        dataTypes.push_back("8A");
1927        names.push_back("MEMBER_URI_TYPE");
1928        dataTypes.push_back("3A");
1929        names.push_back("MEMBER_LOCATION");
1930        dataTypes.push_back(pathTypeName.str());
1931        names.push_back("MEMBER_NAME");
1932        dataTypes.push_back(pathTypeName.str());
1933
1934        groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
1935     }
1936     catch (CCfits::FitsError e)
1937     {
1938         stringstream str;
1939         str << "Could not open or create FITS table GROUPING in  file " << groupName.str() << " reason: " << e.message();
1940         Error(str);
1941         return;
1942     }
1943
1944    //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about.
1945    //use cfitsio routines instead
1946    groupTable->makeThisCurrent();
1947    //create appropriate buffer.
1948    unsigned char* fitsBuffer = new unsigned char[8 + 3 + 2*maxCharLength + 1]; //+1 for trailling character
1949    memset(fitsBuffer, 0, 8 + 3 + 2*maxCharLength + 1);
1950    char* startOfExtension = reinterpret_cast<char*>(fitsBuffer);
1951    char* startOfURI = reinterpret_cast<char*>(&fitsBuffer[8]);
1952    char* startOfLocation = reinterpret_cast<char*>(&fitsBuffer[8 + 3]);
1953    char* startOfName = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]);
1954 //   char* startOfNameVisible = reinterpret_cast<char*>(&fitsBuffer[8 + 3 + 2*maxCharLength]);
1955    sprintf(startOfExtension, "%s", "BINTABLE");
1956    sprintf(startOfURI, "%s", "URL");
1957    int i=1;
1958    for (map<string, vector<string> >::iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++)
1959        for (vector<string>::iterator jt=it->second.begin(); jt != it->second.end(); jt++, i++)
1960        {
1961            strcpy(startOfLocation, it->first.c_str());
1962            strcpy(startOfName, jt->c_str());
1963            if (fDebugIsOn)
1964            {
1965                stringstream str;
1966                str << "Grouping " << it->first << " " << *jt;
1967                Debug(str.str());
1968            }
1969 //           strcpy(startOfNameVisible, jt->c_str());
1970            int status = 0;
1971            fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status);
1972            if (status)
1973            {
1974                stringstream str;
1975                str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status;
1976                Error(str.str());
1977            }
1978        }
1979
1980    filesToGroup.clear();
1981    delete groupFile;
1982}
1983#endif //HAVE_FITS
1984// --------------------------------------------------------------------------
1985//
1986//! Implements the StopRun transition.
1987//! Attempts to close the run file.
1988//! @returns
1989//!        kSM_WaitingRun if success, kSM_FatalError otherwise
1990int DataLogger::StopRunPlease()
1991{
1992
1993    if (fDebugIsOn)
1994    {
1995        Debug("Stopping Run Logging...");   
1996    }
1997    if (!fRunLogFile.is_open() || !fRunReportFile.is_open())
1998        return kSM_FatalError;
1999    if (fRunLogFile.is_open())
2000        fRunLogFile.close();
2001    if (fRunReportFile.is_open())
2002        fRunReportFile.close();
2003#ifdef HAVE_FITS
2004    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2005        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2006        {
2007            if (j->second.runFile.IsOpen())
2008                j->second.runFile.Close();
2009        }
2010    if (fRunFitsFile != NULL)
2011    {
2012        delete fRunFitsFile;
2013        fRunFitsFile = NULL;   
2014        (fNumSubAndFitsData.numOpenFits)--;
2015    }
2016#endif
2017    NotifyOpenedFile("", 0, fOpenedRunFiles);
2018    if (fNumSubAndFitsIsOn)
2019        fNumSubAndFits->updateService();
2020
2021    CreateFitsGrouping(true);
2022
2023    return kSM_WaitingRun;
2024
2025}
2026// --------------------------------------------------------------------------
2027//
2028//! Implements the Stop and Reset transitions.
2029//! Attempts to close any openned file.
2030//! @returns
2031//!     kSM_Ready
2032int DataLogger::GoToReadyPlease()
2033{
2034    if (fDebugIsOn)
2035    {
2036        Debug("Going to the Ready state...");
2037    }   
2038    if (fNightlyLogFile.is_open())
2039        fNightlyLogFile.close();
2040    if (fNightlyReportFile.is_open())
2041        fNightlyReportFile.close();
2042
2043    if (fRunLogFile.is_open())
2044        fRunLogFile.close();
2045    if (fRunReportFile.is_open())
2046        fRunReportFile.close();
2047       
2048#ifdef HAVE_FITS
2049    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2050        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2051        {
2052            if (j->second.nightlyFile.IsOpen())
2053                j->second.nightlyFile.Close();
2054            if (j->second.runFile.IsOpen())
2055                j->second.runFile.Close();
2056        }
2057    if (fRunFitsFile != NULL)
2058    {
2059        delete fRunFitsFile;
2060        fRunFitsFile = NULL;
2061        (fNumSubAndFitsData.numOpenFits)--;
2062    }
2063#endif
2064    if (GetCurrentState() == kSM_Logging)
2065        NotifyOpenedFile("", 0, fOpenedRunFiles);
2066    if (GetCurrentState() == kSM_Logging || 
2067        GetCurrentState() == kSM_WaitingRun || 
2068        GetCurrentState() == kSM_NightlyOpen)
2069    { 
2070        NotifyOpenedFile("", 0, fOpenedNightlyFiles);
2071        if (fNumSubAndFitsIsOn)
2072            fNumSubAndFits->updateService();
2073    }
2074    CreateFitsGrouping(true);
2075    CreateFitsGrouping(false);
2076
2077    return kSM_Ready;
2078}
2079// --------------------------------------------------------------------------
2080//
2081//! Implements the transition towards kSM_WaitingRun
2082//! Does nothing really.
2083//!    @returns
2084//!        kSM_WaitingRun
2085int DataLogger::NightlyToWaitRunPlease()
2086{
2087    if (fDebugIsOn)
2088    {
2089        Debug("Going to Wait Run Number state...");   
2090    }
2091    return kSM_WaitingRun;   
2092}
2093
2094bool DataLogger::SetConfiguration(Configuration& conf)
2095{
2096    fDebugIsOn = conf.Get<bool>("debug");
2097
2098    //Set the block or allow list
2099    fBlackList.clear();
2100    fWhiteList.clear();
2101    if (conf.Has("block"))
2102    {
2103        vector<string> vec = conf.Get<vector<string>>("block");
2104        if (vec.size() != 0)
2105        {
2106            fHasBlackList = true;
2107            if (fDebugIsOn)
2108                Debug("Setting BLOCK list:");
2109        }
2110        for (vector<string>::iterator it = vec.begin(); it != vec.end(); it++)
2111        {
2112            fBlackList.insert(*it);
2113            if (fDebugIsOn)
2114                Debug("                   " + *it);
2115        }
2116    }
2117    if (conf.Has("allow"))
2118    {
2119        vector<string> vec = conf.Get<vector<string>>("allow");
2120        if (vec.size() != 0)
2121        {
2122            fHasWhiteList = true;
2123            if (fDebugIsOn)
2124                Debug("Setting ALLOW list:");
2125        }
2126        for (vector<string>::iterator it=vec.begin(); it != vec.end(); it++)
2127        {
2128            fWhiteList.insert(*it);
2129            if (fDebugIsOn)
2130                Debug("                   " + *it);
2131        }
2132    }
2133    //Set the grouping
2134    if (conf.Has("group"))
2135    {
2136        vector<string> vec = conf.Get<vector<string>>("group");
2137        if (vec.size() != 0)
2138            if (fDebugIsOn)
2139                Debug("Setting GROUPING list:");
2140        for (vector<string>::iterator it=vec.begin(); it != vec.end(); it++)
2141        {
2142            fGrouping.insert(*it);
2143            if (fDebugIsOn)
2144                Debug("                   " + *it);
2145        }
2146    }
2147    return true;
2148}
2149
2150// --------------------------------------------------------------------------
2151
2152int RunDim(Configuration &conf)
2153{
2154    WindowLog wout;
2155
2156    //log.SetWindow(stdscr);
2157    if (conf.Has("log"))
2158        if (!wout.OpenLogFile(conf.Get<string>("log")))
2159            wout << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;
2160
2161    // Start io_service.Run to use the StateMachineImp::Run() loop
2162    // Start io_service.run to only use the commandHandler command detaching
2163    DataLogger logger(wout);
2164    if (!logger.SetConfiguration(conf))
2165        return -1;
2166
2167    logger.Run(true);
2168
2169    return 0;
2170}
2171
2172void RunThread(DataLogger* logger)
2173{
2174    // This is necessary so that the StateMachine Thread can signal the
2175    // Readline thread to exit
2176    logger->Run(true);
2177    Readline::Stop();   
2178}
2179
2180template<class T>
2181int RunShell(Configuration &conf)
2182{
2183    static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);
2184
2185    WindowLog &win  = shell.GetStreamIn();
2186    WindowLog &wout = shell.GetStreamOut();
2187
2188    if (conf.Has("log"))
2189        if (!wout.OpenLogFile(conf.Get<string>("log")))
2190            win << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;
2191
2192    DataLogger logger(wout);
2193    if (!logger.SetConfiguration(conf))
2194        return -1;
2195   
2196    shell.SetReceiver(logger);
2197
2198    boost::thread t(boost::bind(RunThread, &logger));
2199   
2200    shell.Run(); // Run the shell
2201   
2202    logger.Stop();
2203   
2204    //Wait until the StateMachine has finished its thread
2205    //before returning and destroyinng the dim objects which might
2206    //still be in use.
2207    t.join();
2208
2209    return 0;
2210}
2211
2212/*
2213 Extract usage clause(s) [if any] for SYNOPSIS.
2214 Translators: "Usage" and "or" here are patterns (regular expressions) which
2215 are used to match the usage synopsis in program output.  An example from cp
2216 (GNU coreutils) which contains both strings:
2217  Usage: cp [OPTION]... [-T] SOURCE DEST
2218    or:  cp [OPTION]... SOURCE... DIRECTORY
2219    or:  cp [OPTION]... -t DIRECTORY SOURCE...
2220 */
2221void PrintUsage()
2222{
2223    cout << "\n"
2224        "The data logger connects to all available Dim services and "
2225        "writes them to ascii and fits files.\n"
2226        "\n"
2227        "The default is that the program is started without user interaction. "
2228        "All actions are supposed to arrive as DimCommands. Using the -c "
2229        "option, a local shell can be initialized. With h or help a short "
2230        "help message about the usage can be brought to the screen.\n"
2231        "\n"
2232        "Usage: dataLogger [-c type] [OPTIONS]\n"
2233        "  or:  dataLogger [OPTIONS]\n";
2234    cout << endl;
2235
2236}
2237
2238void PrintHelp()
2239{
2240    /* Additional help text which is printed after the configuration
2241     options goes here */
2242    cout <<
2243        "\n"
2244        "The block option has priority over the allow, "
2245        "i.e. if both are present, only the block list is kept. "
2246        "If only a server name or service without its server prefix is given "
2247        "then all the services of that server, or all the services that "
2248        "correspond to the given suffix are ignored or considered.\n"
2249        "\n"
2250        "For example, block=DIS_DNS will skip all the services offered by "
2251        "the DIS_DNS server, while block=SERVICE_LIST will skip all the "
2252        "SERVICE_LIST services offered by any server.\n"
2253        "\n"
2254        "The commands offered by the dataLoger are the following: \n";
2255    cout << setw(20) << DataLogger::fConfigDay << " : specify the path where to put the nightly files\n";
2256    cout << setw(20) << DataLogger::fConfigRun << " : specify the path where to put the run files\n";
2257    cout << setw(20) << DataLogger::fConfigRunNumber << " : specify the run number\n";
2258    cout << setw(20) << DataLogger::fConfigLog << " : log a particular message\n";
2259    cout << setw(20) << DataLogger::fTransStart << " : start the nightly logging\n";
2260    cout << setw(20) << DataLogger::fTransStop << " : stop the nightly logging\n";
2261    cout << setw(20) << DataLogger::fTransStartRun << " : start the run logging\n";
2262    cout << setw(20) << DataLogger::fTransStopRun << " : stop the run logging\n";
2263    cout << setw(20) << DataLogger::fTransReset << " : stop any logging and/or recover from an error state\n";
2264    cout << setw(20) << DataLogger::fTransWait << " : go to the wait for run number state\n";
2265    cout << setw(20) << DataLogger::fPrintCommand << " : print the current state of the logger to the shell\n";
2266    cout << setw(20) << DataLogger::fDebugOnOff << " : turn on or off the debug mode\n";
2267    cout << setw(20) << DataLogger::fStatsPeriod << " : set the periodicity of the statistics. 0 disable them\n";
2268    cout << endl;
2269}
2270
2271void SetupConfiguration(Configuration &conf)
2272{
2273    const string n = conf.GetName()+".log";
2274
2275    po::options_description configp("Program options");
2276    configp.add_options()
2277        ("dns",       var<string>("localhost"),  "Dim nameserver host name (Overwites DIM_DNS_NODE environment variable)")
2278        ("log,l",     var<string>(n), "Write log-file")
2279        ("console,c", var<int>(),     "Use console (0=shell, 1=simple buffered, X=simple unbuffered)")
2280        ;
2281
2282    po::options_description configs("Scheduler options");
2283    configs.add_options()
2284        ("block,b", vars<string>(), "Black-list of services")
2285        ("allow,a", vars<string>(), "White-list of services")
2286        ("debug",   po_bool(),      "Debug mode. Print clear text of received service reports to log-stream")
2287        ("group,g", vars<string>(), "Grouping of services into a single run-Fits")
2288        ;
2289
2290    conf.AddEnv("dns", "DIM_DNS_NODE");
2291
2292    conf.AddOptions(configp);
2293    conf.AddOptions(configs);
2294}
2295
2296int main(int argc, const char* argv[])
2297{
2298    Configuration conf(argv[0]);
2299    conf.SetPrintUsage(PrintUsage);
2300    SetupConfiguration(conf);
2301
2302    po::variables_map vm;
2303    try
2304    {
2305        vm = conf.Parse(argc, argv);
2306    }
2307    catch (exception &e)
2308    {
2309#if BOOST_VERSION > 104000
2310        po::multiple_occurrences *MO = dynamic_cast<po::multiple_occurrences*>(&e);
2311        if (MO)
2312            cout << "Error: " << e.what() << " of '" << MO->get_option_name() << "' option." << endl;
2313        else
2314#endif
2315            cout << "Error: " << e.what() << endl;
2316        cout << endl;
2317
2318        return -1;
2319    }
2320
2321    if (conf.HasPrint())
2322        return -1;
2323
2324    if (conf.HasVersion())
2325    {
2326        FACT::PrintVersion(argv[0]);
2327        return -1;
2328    }
2329
2330    if (conf.HasHelp())
2331    {
2332        PrintHelp();
2333        return -1;
2334    }
2335
2336    Dim::Setup(conf.Get<string>("dns"));
2337
2338//    try
2339    {
2340        // No console access at all
2341        if (!conf.Has("console"))
2342            return RunDim(conf);
2343
2344        // Console access w/ and w/o Dim
2345        if (conf.Get<int>("console")==0)
2346            return RunShell<LocalShell>(conf);
2347        else
2348            return RunShell<LocalConsole>(conf);
2349    }
2350/*    catch (exception& e)
2351    {
2352        cerr << "Exception: " << e.what() << endl;
2353        return -1;
2354    }*/
2355
2356    return 0;
2357}
Note: See TracBrowser for help on using the repository browser.