source: trunk/FACT++/src/dataLogger.cc @ 10968

Last change on this file since 10968 was 10968, checked in by tbretz, 9 years ago
Added some const-qualifiers and did a few minor changes to the layout of the code.
File size: 90.5 KB
Line 
1//****************************************************************
2/** @class DataLogger
3 
4  @brief Logs all message and infos between the services
5 
6  This is the main logging class facility.
7  It derives from StateMachineDim and DimInfoHandler. the first parent is here to enforce
8  a state machine behaviour, while the second one is meant to make the dataLogger receive
9  dim services to which it subscribed from.
10  The possible states and transitions of the machine are:
11  \dot
12  digraph datalogger {
13          node [shape=record, fontname=Helvetica, fontsize=10];
14      e [label="Error" color="red"];
15   r [label="Ready"]
16   d [label="NightlyOpen"]
17   w [label="WaitingRun"]
18      l [label="Logging"]
19   b [label="BadNightlyconfig" color="red"]
20   c [label="BadRunConfig" color="red"]
21 
22  e -> r
23  r -> e
24  r -> d
25  r -> b
26  d -> w
27  d -> r
28  w -> r
29  l -> r
30  l -> w
31  b -> d
32  w -> c
33  w -> l
34  b -> r
35  c -> r
36  c -> l
37  }
38  \enddot
39 */
40 //****************************************************************
41#include "Dim.h"
42#include "Event.h"
43#include "Time.h"
44#include "StateMachineDim.h"
45#include "WindowLog.h"
46#include "Configuration.h"
47#include "ServiceList.h"
48#include "Converter.h"
49#include "MessageImp.h"
50#include "LocalControl.h"
51#include "DimDescriptionService.h"
52
53#include "Description.h"
54
55//#include "DimServiceInfoList.h"
56#include "DimNetwork.h"
57//for getting stat of opened files
58#include <unistd.h>
59//for getting disk free space
60#include <sys/statvfs.h>
61//for getting files sizes
62#include <sys/stat.h>
63
64#include <fstream>
65#include <mutex>
66
67#include <boost/bind.hpp>
68#if BOOST_VERSION < 104400
69#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
70#undef BOOST_HAS_RVALUE_REFS
71#endif
72#endif
73#include <boost/thread.hpp>
74
75#ifdef HAVE_FITS
76#include "Fits.h"
77#endif
78
79//Dim structures
80///Distributes the writing statistics
81struct DataLoggerStats {
82    long sizeWritten;
83    long freeSpace;
84    long writingRate;
85};
86///distributes the number of opened subscriptions and fits files
87struct NumSubAndFitsType {
88    int numSubscriptions;
89    int numOpenFits;
90};
91///distributes which files were opened.
92struct OpenFileToDim {
93    int code;
94    char fileName[FILENAME_MAX];
95};
96
97///Run number record. Used to keep track of which run numbers are still active
98struct RunNumberType {
99#ifdef RUN_LOGS
100    ///the run number log file
101    shared_ptr<ofstream> logFile;
102#endif
103    ///the run number report file
104    shared_ptr<ofstream> reportFile;
105#ifdef HAVE_FITS
106    ///the run number group fits file
107    shared_ptr<CCfits::FITS> runFitsFile;
108#endif
109#ifdef RUN_LOGS
110    ///the log filename
111    string logName;
112#endif
113    ///the report filename
114    string reportName;
115    ///the actual run number
116    uint32_t runNumber;
117    ///the time at which the run number was received
118    Time time;
119    ///list of opened fits used to create the fits grouping when the run ends
120    map<string, vector<string> > openedFits;
121    ///default constructor
122    RunNumberType()
123    {
124#ifdef RUN_LOGS
125        logFile = shared_ptr<ofstream>(new ofstream());
126#endif
127        reportFile = shared_ptr<ofstream>(new ofstream());
128#ifdef HAVE_FITS
129        runFitsFile = shared_ptr<CCfits::FITS>();
130#endif
131        runNumber = 0;
132    }
133    ///default destructor
134    ~RunNumberType()
135    {
136
137    }
138
139    void addServiceToOpenedFits(const string& fileName, const string& serviceName)
140    {
141         //most likely I should add this service name.
142         //the only case for which I should not add it is if a service disapeared, hence the file was closed
143         //and reopened again. Unlikely to happen, but well it may
144
145         if (find(openedFits[fileName].begin(), openedFits[fileName].end(),
146                  serviceName)==openedFits[fileName].end())
147            openedFits[fileName].push_back(serviceName);
148    }
149};
150///Dim subscription type. Stores all the relevant info to handle a Dim subscription
151struct SubscriptionType
152{
153#ifdef HAVE_FITS
154    ///Nightly FITS output file
155    Fits    nightlyFile;
156    ///run-specific FITS output file
157    Fits    runFile;
158#endif
159    ///the actual dimInfo pointer
160    shared_ptr<DimStampedInfo> dimInfo;
161    ///the server
162    string server;
163    ///the service
164    string service;
165    ///the converter for outputting the data according to the format
166    shared_ptr<Converter> fConv;
167    ///the current run number used by this subscription
168    uint32_t runNumber;
169    ///time of the latest received event
170    Time lastReceivedEvent;
171    ///whether or not the fits buffer was allocated already
172    bool fitsBufferAllocated;
173
174    ///Dim info constructor
175    SubscriptionType(DimStampedInfo* info=NULL)
176    {
177        dimInfo = shared_ptr<DimStampedInfo>(info);
178        fConv = shared_ptr<Converter>();
179        runNumber = 0;
180        lastReceivedEvent = Time::None;
181        fitsBufferAllocated = false;
182    }
183
184    ///default destructor
185    ~SubscriptionType()
186    {
187    }
188};
189
190class DataLogger : public StateMachineDim, DimServiceInfoList
191{
192public:
193    /// The list of existing states specific to the DataLogger
194    enum
195    {
196        kSM_NightlyOpen = 20, ///< Nightly file openned and writing
197        kSM_WaitingRun = 30, ///< waiting for the run number to open the run file
198        kSM_Logging = 40, ///< both files openned and writing
199        kSM_BadNightlyConfig = 0x101, ///< the folder specified for Nightly logging does not exist or has bad permissions
200        kSM_BadRunConfig = 0x102, ///<  the folder specified for the run logging does not exist or has wrong permissions or no run number
201        kSM_WriteError = 0x103, ///< Denotes that an error occured while writing a file (text or fits).
202    } localstates_t;
203   
204    DataLogger(ostream &out);
205    ~DataLogger(); 
206
207    bool SetConfiguration(Configuration& conf);
208
209private:
210    /************************************************
211     * MEMBER VARIABLES
212     ************************************************/
213    /// ofstream for the NightlyLogfile
214    ofstream fNightlyLogFile;
215    /// ofstream for the Nightly report file
216    ofstream fNightlyReportFile;
217    /// base path of the Nightlyfile
218    string fNightlyFilePath;
219    ///base path of the run file
220    string fRunFilePath;
221    ///run numbers
222    list<RunNumberType> fRunNumber;
223    ///previous run number. to check if changed while logging
224    int fPreviousRunNumber;
225    ///Current Service Quality
226    int fQuality;
227    ///Modified Julian Date
228    double fMjD;
229    ///for obtaining the name of the existing services
230//    ServiceList fServiceList;
231    typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
232    ///All the services to which we have subscribed to, sorted by server name.
233    SubscriptionsListType fServiceSubscriptions;
234    ///full name of the nightly log file
235    string fFullNightlyLogFileName;
236    ///full name of the nightly report file
237    string fFullNightlyReportFileName;
238    ///variables for computing statistics
239    DataLoggerStats fStatVar;
240    ///variable to track when the statistic were last calculated
241    double fPreviousStatsUpdateTime;
242    double fPreviousOldRunNumberCheck;
243    ///mutex to make sure that the Stats are not accessed while updating
244    mutex fStatsMutex;
245    ///boolean to know whether we should close and reopen daily files or not
246    bool fDailyFileDayChangedAlready;
247public:
248    /***************************************************
249     * STATIC COMMAND NAMES
250     ***************************************************/
251    ///Define all the static names
252    static const char* fConfigDay;
253    static const char* fConfigRun;
254    static const char* fConfigRunNumber;
255    static const char* fConfigLog;
256    static const char* fTransStart;
257    static const char* fTransStop;
258    static const char* fTransStartRun;
259    static const char* fTransStopRun;
260    static const char* fTransReset;
261    static const char* fTransWait;
262    static const char* fRunNumberInfo; ///< This is the name of the dimInfo received to specify the run number. It must be updated once the final name will be defined
263    static const char* fPrintCommand;
264    static const char* fDebugOnOff;
265    static const char* fStatsPeriod;
266    static const char* fStartStopOpenedFiles;
267    static const char* fStartStopNumSubsAndFits;
268private:
269    /***************************************************
270     * DIM INFO HANDLER
271     ***************************************************/
272    //overloading of DIM's infoHandler function
273    void infoHandler(); 
274
275    /***************************************************
276     * TRANSITION FUNCTIONS
277     ***************************************************/
278    ///Reporting method for the services info received
279    void ReportPlease(DimInfo* I, SubscriptionType& sub); 
280    ///Configuration of the nightly file path
281    int ConfigureNightlyFileName(const Event& evt); 
282    ///Configuration fo the file name
283    int ConfigureRunFileName(const Event& evt); 
284    ///DEPREC - configuration of the run number
285    int ConfigureRunNumber(const Event& evt); 
286    ///print the current state of the dataLogger
287    int PrintStatePlease(const Event& evt);
288    ///checks whether or not the current info being treated is a run number
289    void CheckForRunNumber(DimInfo* I);
290    /// start transition
291    int StartPlease(); 
292    ///from waiting to logging transition
293    int StartRunPlease(); 
294    /// from logging to waiting transition
295    int StopRunPlease(); 
296    ///stop and reset transition
297    int GoToReadyPlease(); 
298    ///from NightlyOpen to waiting transition
299    int NightlyToWaitRunPlease(); 
300    /// from writing to error
301    std::string SetCurrentState(int state, const char *txt="", const std::string &cmd="");
302#ifdef HAVE_FITS
303    ///Open fits files
304    void OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber);
305    ///Write data to FITS files
306    void WriteToFITS(SubscriptionType& sub);
307    ///Allocate the buffers required for fits
308    void AllocateFITSBuffers(SubscriptionType& sub);
309#endif//has_fits
310
311    /***************************************
312     * DIM SERVICES PROVIDED BY THE DATA LOGGER
313     ***************************************/
314    ///monitoring notification loop
315    void ServicesMonitoring();
316    inline void NotifyOpenedFile(const string &name, int type, DimDescribedService* service);
317    ///services notification thread
318    boost::thread fMonitoringThread;
319    ///end of the monitoring
320    bool fContinueMonitoring;
321    ///stores the size of each file that is or was open
322    map<string, long> fFileSizesMap;
323    ///total size of the opened files BEFORE they were opened by the logger
324    long fBaseSizeNightly;
325    long fPreviousSize;
326    long fBaseSizeRun;
327    ///Service for opened files
328    DimDescribedService* fOpenedNightlyFiles;
329    DimDescribedService* fOpenedRunFiles;
330    DimDescribedService* fNumSubAndFits;
331    NumSubAndFitsType fNumSubAndFitsData;
332    ///Small function for calculating the total size written so far
333    bool calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting);
334
335    /***************************************************
336     * DATA LOGGER's CONFIGURATION STUFF
337     ***************************************************/
338    ///black/white listing
339    set<string> fBlackList;
340    set<string> fWhiteList;
341    ///list of services to be grouped
342    set<string> fGrouping;
343    ///configuration flags
344    bool fDebugIsOn;
345    float fStatsPeriodDuration;
346    bool fOpenedFilesIsOn;
347    bool fNumSubAndFitsIsOn;
348    //functions for controlling the services behavior
349    int SetDebugOnOff(const Event& evt);
350    int SetStatsPeriod(const Event& evt);
351    int SetOpenedFilesOnOff(const Event& evt);
352    int SetNumSubsAndFitsOnOff(const Event& evt);
353
354    ///boolean to prevent DIM update while desctructing the dataLogger
355    bool fDestructing;   
356    /***************************************************
357     * UTILITIES
358     ***************************************************/
359    ///vectors to keep track of opened Fits files, for grouping purposes.
360    map<string, vector<string> > fOpenedNightlyFits;
361    ///creates a group fits file based on a list of files to be grouped
362    void CreateFitsGrouping(map<string, vector<string> >& filesToGroup, int runNumber);
363    ///Open the relevant text files related to a particular run
364    int OpenRunFile(RunNumberType& run);
365    ///add a new run number
366    void AddNewRunNumber(int64_t newRun, Time time);
367    ///removes the oldest run number, and close the relevant files.
368    void RemoveOldestRunNumber();
369    ///retrieves the size of a file
370    off_t GetFileSize(const string&);
371    ///Get the digits of year, month and day for filenames and paths
372    void GetYearMonthDayForFiles(unsigned short& year, unsigned short& month, unsigned short& day);
373    ///Appends the relevant year month day to a given path
374    void AppendYearMonthDaytoPath(string& path);
375    ///Form the files path
376    string CompileFileName(const string &path, const string &service, const string & extension, const Time &time=Time());
377    ///Form the files path
378    string CompileFileName(const string &path, uint32_t run, const string &service, const string & extension, const Time &time=Time());
379    ///Check whether service is in black and/or white list
380    bool ShouldSubscribe(const string& server, const string& service);
381    ///Subscribe to a given server and service
382    DimStampedInfo* SubscribeToPlease(const string& server, const string& service);
383    ///Open a text file and checks for ofstream status
384    void OpenTextFilePlease(ofstream& stream, const string& name);
385    ///Check if a dir is . and returns the actual string corresponding to .
386//    string CheckIfDirIsDot(const string& dir);
387    ///Remembers the size of newly opened files. for statistic purposes
388    bool RememberFileOrigSizePlease(string& fileName, bool nightly);
389    ///Checks if the input osftream is in error state, and if so close it.
390    void CheckForOfstreamError(ofstream& out);
391    ///Checks if a given path exist
392    bool DoesPathExist(string path);
393public:
394    ///Create a given directory
395    bool CreateDirectory(string path);
396    /***************************************************
397    * INHERITED FROM DIMSERVICEINFOLIST
398    ***************************************************/
399    ///Add a new service subscription
400    void AddService(const string&, const string&, const string&, bool);
401    ///Remove a given service subscription
402    void RemoveService(const string&, const string&, bool);
403    ///Remove all the services associated with a given server
404    void RemoveAllServices(const string&);
405}; //DataLogger
406
407// --------------------------------------------------------------------------
408//
409//! Check if a given path exists
410//! @param path the path to be checked
411//! @return whether or not the creation has been successfull
412//
413bool DataLogger::CreateDirectory(string path)
414{
415    //remove last '/', if present
416    if (path[path.size()-1] == '/')
417        path = path.substr(0, path.size()-1);
418
419    //create boost path
420    const boost::filesystem::path fullPath =  boost::filesystem::system_complete(boost::filesystem::path(path));
421
422    //if path does not exist, check if upper levels exist already
423    if (boost::filesystem::exists(fullPath))
424    {
425        //if path already exist, make sure it does not designate a file (filenames cannot be checked if they do not exist)
426        if (boost::filesystem::is_directory(fullPath))
427            return true;
428
429        Error("Path to be created contains a file name: '" + path + "'");
430        return false;
431    }
432
433    if (path.size() <= 1)
434    {//we're hitting "/", which SHOULD have existed...
435        Error("Something unexpected happened while creating a path");
436    }
437    CreateDirectory(path.substr(0, path.find_last_of('/')));
438
439    //path does not exist, and upper level have been created already by recusrion.
440    const mode_t rightsMask = S_IRWXU | S_IXGRP | S_IRGRP | S_IXOTH | S_IROTH; //everybody read, owner writes
441
442    const int returnValue = mkdir(path.c_str(), rightsMask);
443
444    if (returnValue != 0)
445    {
446        ostringstream str;
447        str << "Could not create directory " << path << " mkdir error code: " << errno;
448        Error(str.str());
449        return false;
450    }
451
452    return true;
453}
454// --------------------------------------------------------------------------
455//
456//! Check if a given path exists
457//! @param path the path to be checked
458//! @return whether or not the given path exists
459//
460bool DataLogger::DoesPathExist(string path)
461{
462    const boost::filesystem::path fullPath = boost::filesystem::system_complete(boost::filesystem::path(path));
463
464    if (!boost::filesystem::exists(fullPath))
465       return false;
466
467    if (!boost::filesystem::is_directory(fullPath))
468    {
469        Error("Path given for checking '" + path + "' designate a file name. Please provide a path name only");
470        return false;
471    }
472
473    if (access(path.c_str(), R_OK|W_OK|X_OK) != 0)
474    {
475        Error("Missing read, write or execute permissions on directory '" + path + "'");
476        return false;
477    }
478
479    return true;
480}
481// --------------------------------------------------------------------------
482//
483//! Add a new service subscription
484//! @param server the server for which the subscription should be created
485//! @param service the service for which the subscription should be created
486//! @param isCmd whether this is a Dim Command or not. Commands are not logged
487//
488void DataLogger::AddService(const string& server, const string& service, const string&, bool isCmd)
489{
490         //dataLogger does not subscribe to commands
491    if (isCmd)
492        return;
493
494    //check the given subscription against black and white lists
495    if (!ShouldSubscribe(server, service))
496        return;
497
498    map<string, SubscriptionType> &list = fServiceSubscriptions[server];
499
500    if (list.find(service) != list.end())
501    {
502        Error("Service " + server + "/" + service + " is already in the dataLogger's list. ignoring its update.");
503        return;
504    }
505
506    list[service].dimInfo = shared_ptr<DimStampedInfo>(SubscribeToPlease(server, service));
507    list[service].server  = server;
508    list[service].service = service;
509    fNumSubAndFitsData.numSubscriptions++;
510    if (fDebugIsOn)
511        Debug("Added subscription to " + server + "/" + service);
512}
513// --------------------------------------------------------------------------
514//
515//! Remove a given service subscription
516//! @param server the server for which the subscription should be removed
517//! @param service the service that should be removed
518//! @param isCmd whether or not this is a command
519//
520void DataLogger::RemoveService(const string& server, const string& service, bool isCmd)
521{
522    if (isCmd)
523        return;
524    if (fServiceSubscriptions[server].erase(service) != 1)
525    {
526        ostringstream str;
527        str << "Subscription " << server << "/" << service << "could not be removed as it is not present";
528        Error(str.str());
529        return;
530    }
531    fNumSubAndFitsData.numSubscriptions--;
532    if (fDebugIsOn)
533    {
534        Debug("Removed subscription to " + server + "/" + service);
535    }
536}
537// --------------------------------------------------------------------------
538//
539//! Remove all the services associated with a given server
540//! @param server the server for which all the services should be removed
541//
542void DataLogger::RemoveAllServices(const string& server)
543{
544    fNumSubAndFitsData.numSubscriptions -= fServiceSubscriptions[server].size();
545    fServiceSubscriptions[server].clear();
546    if (fDebugIsOn)
547    {
548        Debug("Removed all subscriptions to " + server + "/");
549    }
550}
551// --------------------------------------------------------------------------
552//
553//! Checks if the given ofstream is in error state and if so, close it
554//! @param out the ofstream that should be checked
555//
556void DataLogger::CheckForOfstreamError(ofstream& out)
557{
558    if (!out.good())
559    {
560        Error("An error occured while writing to a text file. Closing it");
561        if (out.is_open())
562            out.close();
563        SetCurrentState(kSM_WriteError);
564    }
565}
566// --------------------------------------------------------------------------
567//
568//! Checks the size on disk of a given size, and remembers it in the relevant member variable
569//! @param fileName the file for which the size on disk should be retrieved
570//! @param nightly whether this is a run or nightly file, so that its size is added to the correct member variable
571//
572bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly)
573{
574    //get the size of the file we're about to open
575    if (fFileSizesMap.find(fileName) != fFileSizesMap.end())
576        return false;
577
578    if (nightly)
579        fBaseSizeNightly += GetFileSize(fileName);
580    else
581        fBaseSizeRun += GetFileSize(fileName);
582    fFileSizesMap[fileName] = 0;
583    return true;
584}
585
586// --------------------------------------------------------------------------
587//
588//! Open a text file and checks for error code
589//! @param stream the ofstream for which the file should be opened
590//! @name the file name
591//
592void DataLogger::OpenTextFilePlease(ofstream& stream, const string& name)
593{
594    errno = 0;
595    stream.open(name.c_str(), ios_base::out | ios_base::app);
596    if (!stream)
597    {
598        ostringstream str;
599        str << "Open file " << name << ": " << strerror(errno) << " (errno=" << errno << ")";
600        Error(str);
601    }
602}
603// --------------------------------------------------------------------------
604//
605//! Create a new dim subscription to a given server and service
606//! @param server the server name
607//! @param service the service name
608//
609DimStampedInfo* DataLogger::SubscribeToPlease(const string& server, const string& service)
610{
611    if (fDebugIsOn)
612    {
613        ostringstream str;
614        str << "Subscribing to service " << server << "/" << service;
615        Debug(str);
616    }
617    return new DimStampedInfo((server + "/" + service).c_str(), const_cast<char*>(""), this);
618}
619// --------------------------------------------------------------------------
620//
621//! Check whether a service should be subscribed to, based on the black/white list entries
622//! @param server the server name associated with the service being checked
623//! @param service the service name associated with the service being checked
624//
625bool DataLogger::ShouldSubscribe(const string& server, const string& service)
626{
627    if (fWhiteList.size()>0 &&
628        (fWhiteList.find(server + "/") == fWhiteList.end()) &&
629        (fWhiteList.find(server + "/" + service) == fWhiteList.end()) &&
630        (fWhiteList.find("/" + service) == fWhiteList.end()))
631        return false;
632
633    if ((fBlackList.find(server + "/") != fBlackList.end()) ||
634         (fBlackList.find(server + "/" + service) != fBlackList.end()) ||
635         (fBlackList.find("/" + service) != fBlackList.end()))
636        return false;
637
638    return true;
639}
640// --------------------------------------------------------------------------
641//
642//! Compiles a file name
643//! @param path the base path where to put the file
644//! @param time the time at which the file is created
645//! @param service the service name, if any
646//! @param extension the extension to add, if any
647//
648string DataLogger::CompileFileName(const string &path, const string &service, const string & extension, const Time &time)
649{
650       ostringstream str;
651       //calculate time suitable for naming files.
652       const Time ftime(time-boost::posix_time::time_duration(12,0,0));
653
654       //output it
655       str << path << Time::fmt("/%Y/%m/%d") << ftime;
656
657       //check if target directory exist
658       if (!DoesPathExist(str.str()))
659           CreateDirectory(str.str());
660
661       //output base of file name
662       str << Time::fmt("/%Y_%m_%d") << ftime;
663
664       //output service name
665       if (!service.empty())
666           str  << "_" << service;
667
668       //output appropriate extension
669       if (!extension.empty())
670           str << "." << extension;
671
672       return str.str();
673}
674// --------------------------------------------------------------------------
675//
676//! Compiles a file name
677//! @param path the base path where to put the file
678//! @param time the time at which the file is created
679//! @param run the run number
680//! @param service the service name, if any
681//! @param extension the extension to add, if any
682//
683string DataLogger::CompileFileName(const string &path, uint32_t run, const string &service, const string & extension, const Time &time)
684{
685       ostringstream str;
686       //calculate suitable time for naming files and output it
687       str << path << Time::fmt("/%Y/%m/%d") << (time-boost::posix_time::time_duration(12,0,0));
688
689       //check if target directory exist
690       if (!DoesPathExist(str.str()))
691           CreateDirectory(str.str());
692
693       //output base of file name
694       str << '/' << setfill('0') << setw(8) << run;
695
696       //output service name
697       if (!service.empty())
698           str  << "_" << service;
699
700       //output appropriate extension
701       if (!extension.empty())
702           str << "." << extension;
703       return str.str();
704}
705
706// --------------------------------------------------------------------------
707//
708//!retrieves the size on disk of a file
709//! @param fileName the full file name for which the size on disk should be retrieved
710//! @return the size of the file on disk, in bytes. 0 if the file does not exist or if an error occured
711//
712off_t DataLogger::GetFileSize(const string& fileName)
713{
714    errno = 0;
715    struct stat st;
716    if (!stat(fileName.c_str(), &st))
717        return st.st_size;
718
719    if (errno != 0 && errno != 2)//ignoring error #2: no such file or directory is not an error for new files
720    {
721        ostringstream str;
722        str << "Unable to stat " << fileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
723        Error(str);
724    }
725
726    return 0;
727}
728// --------------------------------------------------------------------------
729//
730//! Removes the oldest run number and closes the fits files that should be closed
731//! Also creates the fits grouping file
732//
733void DataLogger::RemoveOldestRunNumber()
734{
735    if (fDebugIsOn)
736    {
737        ostringstream str;
738        str << "Removing run number " << fRunNumber.front().runNumber;
739        Debug(str);
740    }
741    CreateFitsGrouping(fRunNumber.front().openedFits, fRunNumber.front().runNumber);
742
743    //crawl through the subscriptions to see if there are still corresponding fits files opened.
744    for (SubscriptionsListType::iterator x=fServiceSubscriptions.begin();
745         x!=fServiceSubscriptions.end(); x++)
746        for (map<string, SubscriptionType>::iterator y=x->second.begin();
747             y!=x->second.end(); y++)
748            if (y->second.runFile.fRunNumber == fRunNumber.front().runNumber && y->second.runFile.IsOpen())
749            {
750                if (fDebugIsOn)
751                {
752                    ostringstream str;
753                    str << "Closing Fits run file " << y->second.runFile.fFileName;
754                    Debug(str);
755                }
756                y->second.runFile.Close();
757            }
758    //if a grouping file is on, decrease the number of opened fits manually
759    if (fRunNumber.front().runFitsFile)
760        (fNumSubAndFitsData.numOpenFits)--;
761    //remove the entry
762    fRunNumber.pop_front();
763}
764
765// --------------------------------------------------------------------------
766//
767//! Calculate the total number of written bytes since the logger was started
768//! @param statVar the data structure that should be updated
769//! @param shouldWarn whether or not error messages should be outputted
770//! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information
771//
772bool DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting)
773{
774    if (!isPrinting)
775        fStatsMutex.lock();
776#ifdef HAVE_FITS
777    if (isPrinting)
778    {
779        ostringstream str;
780        str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:";
781        Message(str);
782    }
783
784    ///TODO the grouping file is dealt with several times. This should not be a problem but well, better to fix it I guess.
785    for (SubscriptionsListType::const_iterator x=fServiceSubscriptions.begin();
786         x!=fServiceSubscriptions.end(); x++)
787    {
788        for (map<string, SubscriptionType>::const_iterator y=x->second.begin();
789             y!=x->second.end(); y++)
790        {
791            if (y->second.runFile.IsOpen())
792            {
793                fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();
794                if (isPrinting)
795                    Message("-> "+y->second.runFile.fFileName);
796            }
797            if (y->second.nightlyFile.IsOpen())
798            {
799                fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();
800                if (isPrinting)
801                    Message("-> "+y->second.nightlyFile.fFileName);
802            }
803        }
804    }
805#else
806    if (isPrinting)
807        Message("FITS output disabled at compilation");
808#endif
809    //gather log and report files sizes on disk
810    if (fNightlyLogFile.is_open())
811        fFileSizesMap[fFullNightlyLogFileName] = GetFileSize(fFullNightlyLogFileName);
812    if (fNightlyReportFile.is_open())
813        fFileSizesMap[fFullNightlyReportFileName] = GetFileSize(fFullNightlyReportFileName);
814    for (list<RunNumberType>::iterator it = fRunNumber.begin(); it != fRunNumber.end(); it++)
815    {
816        if (it->reportFile->is_open())
817            fFileSizesMap[it->reportName] = GetFileSize(it->reportName);
818#ifdef RUN_LOGS
819        if (it->logFile->is_open())
820            fFileSizesMap[it->logName] = GetFileSize(it->logName);
821#endif
822    }
823
824    bool shouldWarn = false;
825    struct statvfs vfs;
826    if (!statvfs(fNightlyFilePath.c_str(), &vfs))
827        statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
828    else
829    {
830        ostringstream str;
831        str << "Unable to retrieve stats for " << fNightlyFilePath << ". Reason: " << strerror(errno) << " [" << errno << "]";
832        if (!shouldWarn)
833            Error(str);
834        statVar.freeSpace = -1;
835    }
836    //sum up all the file sizes. past and present
837    statVar.sizeWritten = 0;
838    for (map<string, long>::const_iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end();  it++)
839        statVar.sizeWritten += it->second;
840    statVar.sizeWritten -= fBaseSizeNightly;
841    statVar.sizeWritten -= fBaseSizeRun;
842
843    if (!isPrinting)
844        fStatsMutex.unlock();
845
846    return shouldWarn;
847}
848
849//static members initialization
850//since I do not check the transition/config names any longer, indeed maybe these could be hard-coded... but who knows what will happen in the future ?
851const char* DataLogger::fConfigDay = "CONFIG_DAY";
852const char* DataLogger::fConfigRun = "CONFIG_RUN";
853const char* DataLogger::fConfigRunNumber = "CONFIG_RUN_NUMBER";
854const char* DataLogger::fConfigLog = "LOG";
855const char* DataLogger::fTransStart = "START";
856const char* DataLogger::fTransStop = "STOP";
857const char* DataLogger::fTransStartRun = "START_RUN";
858const char* DataLogger::fTransStopRun = "STOP_RUN";
859const char* DataLogger::fTransReset = "RESET";
860const char* DataLogger::fTransWait = "WAIT_RUN_NUMBER";
861const char* DataLogger::fRunNumberInfo = "RUN_NUMBER";
862const char* DataLogger::fPrintCommand = "PRINT";
863const char* DataLogger::fDebugOnOff = "DEBUG";
864const char* DataLogger::fStatsPeriod = "STATS_PERIOD";
865const char* DataLogger::fStartStopOpenedFiles = "OPENED_FILES_SRVC";
866const char* DataLogger::fStartStopNumSubsAndFits = "NUM_SUBS_SRVC";
867
868// --------------------------------------------------------------------------
869//
870//! Monitor the number of opened files and total size written, and distributes this data through a Dim service
871//
872//
873void DataLogger::ServicesMonitoring()
874{
875       struct statvfs vfs;
876        if (!statvfs(fNightlyFilePath.c_str(), &vfs))
877            fStatVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
878        else
879            fStatVar.freeSpace = -1;
880
881        DimDescribedService srvc ("DATA_LOGGER/STATS", "X:3", fStatVar, "Add description here");
882        fPreviousSize = 0;
883        //loop-wait for broadcast
884        while (fContinueMonitoring)
885        {
886            if (fStatsPeriodDuration == 0.0f)
887            {
888                sleep(0.1f);
889                continue;
890            }
891
892            sleep(fStatsPeriodDuration);
893
894
895            fStatsMutex.lock();
896
897            if (fStatVar.writingRate != 0) //if data has been written
898            {
899                srvc.updateService();
900
901                if(fDebugIsOn)
902                {
903                    ostringstream str;
904                    str << "Size written: " << fStatVar.sizeWritten/1000 << " kB; writing rate: ";
905                    str << fStatVar.writingRate/1000 << " kB/s; free space: ";
906                    str << fStatVar.freeSpace/(1000*1000) << " MB";
907                    Debug(str);
908                }
909            }
910            fStatsMutex.unlock();
911        }
912}
913
914// --------------------------------------------------------------------------
915//
916//! Default constructor. The name of the machine is given DATA_LOGGER
917//! and the state is set to kSM_Ready at the end of the function.
918//
919//!Setup the allows states, configs and transitions for the data logger
920//
921DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER")
922{
923    //initialize member data
924    fNightlyFilePath = ".";
925    fRunFilePath = ".";
926
927    //Give a name to this machine's specific states
928    AddStateName(kSM_NightlyOpen,      "NightlyFileOpen",  "The summary files for the night are open.");
929    AddStateName(kSM_WaitingRun,       "WaitForRun",       "The summary files for the night are open and we wait for a run to be started.");
930    AddStateName(kSM_Logging,          "Logging",          "The summary files for the night and the files for a single run are open.");
931    AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "The folder for the nighly summary files is invalid.");
932    AddStateName(kSM_BadRunConfig,     "ErrRunFolder",     "The folder for the run files is invalid.");
933    AddStateName(kSM_WriteError, "ErrWrite", "An error occured while writing to a file.");
934
935    /*Add the possible transitions for this machine*/
936    AddEvent(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig)
937            (boost::bind(&DataLogger::StartPlease, this))
938            ("Start the nightly logging. Nightly file location must be specified already");
939
940    AddEvent(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging, kSM_WriteError)
941            (boost::bind(&DataLogger::GoToReadyPlease, this))
942            ("Stop all data logging, close all files.");
943
944    AddEvent(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig)
945            (boost::bind(&DataLogger::StartRunPlease, this))
946            ("Start the run logging. Run file location must be specified already.");
947
948    AddEvent(kSM_WaitingRun, fTransStopRun, kSM_Logging)
949            (boost::bind(&DataLogger::StopRunPlease, this))
950            ("Wait for a run to be started, open run-files as soon as a run number arrives.");
951
952    AddEvent(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_WriteError)
953            (boost::bind(&DataLogger::GoToReadyPlease, this))
954            ("Transition to exit error states. Closes the nightly file if already opened.");
955
956    AddEvent(kSM_WaitingRun, fTransWait, kSM_NightlyOpen)
957            (boost::bind(&DataLogger::NightlyToWaitRunPlease, this));
958
959    /*Add the possible configurations for this machine*/
960    AddEvent(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig)
961            (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1))
962            ("Configure the folder for the nightly files."
963             "|Path[string]:Absolute or relative path name where the nightly files should be stored.");
964
965    AddEvent(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
966            (boost::bind(&DataLogger::ConfigureRunFileName, this, _1))
967            ("Configure the folder for the run files."
968             "|Path[string]:Absolute or relative path name where the run files should be stored.");
969
970    AddEvent(fConfigRunNumber, "X", kSM_Ready, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig, kSM_Logging)
971            (boost::bind(&DataLogger::ConfigureRunNumber, this, _1))
972            ("Configure the run number. Cannot be done in logging state");
973
974     //Provide a print command
975     ostringstream str;
976     str <<     kSM_Ready << " " << kSM_NightlyOpen << " " << kSM_WaitingRun << " " << kSM_Logging << " " << kSM_BadNightlyConfig << " " << kSM_WriteError;
977     str << " " << kSM_BadRunConfig;
978     AddEvent(fPrintCommand, str.str().c_str(), "")
979             (boost::bind(&DataLogger::PrintStatePlease, this, _1))
980             ("Print information about the internal status of the data logger.");
981
982     OpenFileToDim fToDim;
983     fToDim.code = 0;
984     fToDim.fileName[0] = '\0';
985
986     fOpenedNightlyFiles = new DimDescribedService(GetName() + "/FILENAME_NIGHTLY", "I:1;C", fToDim,
987                               "Path and base name which is used to compile the filenames for the nightly files."
988                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
989                               "|Name[string]:path and base file name");
990
991     fOpenedRunFiles = new DimDescribedService(GetName() + "/FILENAME_RUN", "I:1;C", fToDim,
992                               "Path and base name which is used to compile the filenames for the run files."
993                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
994                               "|Name[string]:path and base file name");
995
996     fNumSubAndFitsData.numSubscriptions = 0;
997     fNumSubAndFitsData.numOpenFits = 0;
998     fNumSubAndFits = new DimDescribedService(GetName() + "/NUM_SUBS", "I:2", fNumSubAndFitsData,
999                               "Shows number of services to which the data logger is currently subscribed and the total number of open files."
1000                               "|Subscriptions[int]:number of dim services to which the data logger is currently subscribed."
1001                               "|NumOpenFiles[int]:number of files currently open by the data logger");
1002
1003     //services parameters
1004     fDebugIsOn = false;
1005     fStatsPeriodDuration = 1.0f;
1006     fOpenedFilesIsOn = true;
1007     fNumSubAndFitsIsOn = true;
1008
1009     //provide services control commands
1010     AddEvent(fDebugOnOff, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1011             (boost::bind(&DataLogger::SetDebugOnOff, this, _1))
1012             ("Switch debug mode on off. Debug mode prints ifnormation about every service written to a file."
1013              "|Enable[bool]:Enable of disable debug mode (yes/no).");
1014
1015     AddEvent(fStatsPeriod, "F", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1016             (boost::bind(&DataLogger::SetStatsPeriod, this, _1))
1017             ("Interval in which the data-logger statitistics service (STATS) is updated."
1018              "Interval[s]:Floating point value in seconds.");
1019
1020     AddEvent(fStartStopOpenedFiles, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1021              (boost::bind(&DataLogger::SetOpenedFilesOnOff ,this, _1))
1022              ("Switch off the service which distributes information about the open files.");
1023
1024     AddEvent(fStartStopNumSubsAndFits, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1025             (boost::bind(&DataLogger::SetNumSubsAndFitsOnOff, this, _1))
1026             ("Switch off the service which distributes information about the number of subscriptions and open files.");
1027
1028     fDestructing = false;
1029
1030     //start the monitoring service
1031     fStatVar.sizeWritten = 0;
1032     fStatVar.freeSpace = 0;
1033     fStatVar.writingRate = 0;
1034     fPreviousStatsUpdateTime = Time().Mjd();
1035     fPreviousOldRunNumberCheck = Time().Mjd();
1036     fContinueMonitoring = true;
1037     fMonitoringThread = boost::thread(boost::bind(&DataLogger::ServicesMonitoring, this));
1038     fBaseSizeNightly = 0;
1039     fBaseSizeRun = 0;
1040     fDailyFileDayChangedAlready = true;
1041     if(fDebugIsOn)
1042     {
1043         Debug("DataLogger Init Done.");
1044     }
1045}
1046
1047// --------------------------------------------------------------------------
1048//
1049//! Destructor
1050//
1051DataLogger::~DataLogger()
1052{
1053    if (fDebugIsOn)
1054    {
1055        Debug("DataLogger destruction starts");   
1056    }
1057    fDestructing = true;
1058    //first let's go to the ready state
1059    GoToReadyPlease(); 
1060    //release the services subscriptions
1061    fServiceSubscriptions.clear();
1062    //exit the monitoring loop
1063    fContinueMonitoring = false;
1064
1065    fMonitoringThread.join();
1066    //clear any remaining run number (should remain only one)
1067     while (fRunNumber.size() > 0)
1068     {
1069         RemoveOldestRunNumber();
1070     }
1071
1072    delete fOpenedNightlyFiles;
1073    delete fOpenedRunFiles;
1074    delete fNumSubAndFits;
1075
1076    if (fDebugIsOn)
1077    {
1078        Debug("DataLogger desctruction ends");   
1079    }
1080}
1081
1082// --------------------------------------------------------------------------
1083//
1084//! Inherited from DimInfo. Handles all the Infos to which we subscribed, and log them
1085//
1086void DataLogger::infoHandler()
1087{
1088    // Make sure getTimestamp is called _before_ getTimestampMillisecs
1089    if (fDestructing)
1090        return;
1091
1092    DimInfo* I = getInfo();
1093
1094    if (I==NULL)
1095        return;
1096    //check if the service pointer corresponds to something that we subscribed to
1097    //this is a fix for a bug that provides bad Infos when a server starts
1098    bool found = false;
1099    SubscriptionsListType::iterator x;
1100    map<string, SubscriptionType>::iterator y;
1101     for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
1102    {//find current service is subscriptions
1103        for (y=x->second.begin(); y!=x->second.end();y++)
1104            if ((y->second.dimInfo).get() == I)
1105            {
1106                found = true;   
1107                break;
1108            }
1109        if (found)
1110            break;
1111    }
1112    if (!found)
1113    {
1114        DimServiceInfoList::infoHandler();
1115        return;
1116    }
1117    if (I->getSize() <= 0)
1118        return;
1119
1120        // Make sure that getTimestampMillisecs is NEVER called before
1121        // getTimestamp is properly called
1122        // check that the message has been updated by something, i.e. must be different from its initial value
1123    if (I->getTimestamp() == 0)
1124        return;
1125
1126    CheckForRunNumber(I);
1127
1128    ReportPlease(I, y->second);
1129
1130    //update the fits files sizes
1131    const Time cTime = Time();
1132    if ((fStatsPeriodDuration != 0) && ((cTime.Mjd() - fPreviousStatsUpdateTime)*24*60*60 > fStatsPeriodDuration))
1133    {
1134        calculateTotalSizeWritten(fStatVar, false);
1135        fStatVar.writingRate = (fStatVar.sizeWritten - fPreviousSize)/((cTime.Mjd() - fPreviousStatsUpdateTime)*24*60*60);
1136        fPreviousSize = fStatVar.sizeWritten;
1137        fPreviousStatsUpdateTime = cTime.Mjd();
1138    }
1139    if ((cTime.Mjd() - fPreviousOldRunNumberCheck)*24*60*60 > 10.0)
1140    {
1141        while (fRunNumber.size() > 1 && (cTime - fRunNumber.back().time) > boost::posix_time::time_duration(0,0,10,0))
1142        {
1143             RemoveOldestRunNumber();
1144        }
1145        fPreviousOldRunNumberCheck = cTime.Mjd();
1146    }
1147}
1148// --------------------------------------------------------------------------
1149//
1150//! Open the text files associated with the given run number
1151//! @param run the run number to be dealt with
1152//
1153int DataLogger::OpenRunFile(RunNumberType& run)
1154{
1155#ifdef RUN_LOGS
1156    if (run.logFile->is_open())
1157    {
1158        ostringstream str;
1159        str << "Log file " << run.logName << " was already open when trying to open it in OpenRunFile";
1160        Error(str);
1161        return -1;
1162    }
1163    run.logName = CompileFileName(fRunFilePath, run.runNumber, "", "log");
1164
1165    errno = 0;
1166    run.logFile->open(run.logName.c_str(), ios_base::out | ios_base::app);
1167    if (errno != 0)
1168    {
1169        ostringstream str;
1170        str << "Unable to open log-file " << run.logName << ": " << strerror(errno) << " (errno=" << errno << ")";
1171        Error(str);
1172    }
1173#endif
1174    //open report file
1175    run.reportName = CompileFileName(fRunFilePath, run.runNumber, "", "rep");
1176    if (run.reportFile->is_open())
1177    {
1178        ostringstream str;
1179        str << "Report file " << run.reportName << " was already open when trying to open it in OpenRunFile";
1180        Error(str);
1181        return -1;
1182    }
1183
1184    errno = 0;
1185    run.reportFile->open(run.reportName.c_str(), ios_base::out | ios_base::app);
1186    if (errno != 0)
1187    {
1188        ostringstream str;
1189        str << "Unable to open rep-file " << run.reportName << ": " << strerror(errno) << " (errno=" << errno << ")";
1190        Error(str);
1191    }
1192
1193#ifdef RUN_LOGS
1194    if (!run.logFile->is_open() || !run.reportFile->is_open())
1195#else
1196    if (!run.reportFile->is_open())
1197#endif
1198    {
1199        ostringstream str;
1200        str << "Something went wrong while openning files ";
1201#ifdef RUN_LOGS
1202        str << run.logName << " and ";
1203#endif
1204        str << run.reportName;
1205        Error(str);
1206        return -1;
1207    }
1208    //get the size of the newly opened file.
1209#ifdef RUN_LOGS
1210    RememberFileOrigSizePlease(run.logName, false);
1211#endif
1212    RememberFileOrigSizePlease(run.reportName, false);
1213
1214    //TODO this notification scheme might be messed up now.... fix it !
1215    const string baseFileName = CompileFileName(fRunFilePath, run.runNumber, "", "");
1216    NotifyOpenedFile(baseFileName, 3, fOpenedRunFiles);
1217    run.openedFits.clear();
1218    return 0;
1219}
1220// --------------------------------------------------------------------------
1221//
1222//! Add a new active run number
1223//! @param newRun the new run number
1224//! @param time the time at which the new run number was issued
1225//
1226void DataLogger::AddNewRunNumber(int64_t newRun, Time time)
1227{
1228
1229    if (newRun > 0xffffffff)
1230    {
1231        Error("New run number too large, out of range. Ignoring.");
1232        return;
1233    }
1234    if (fDebugIsOn)
1235    {
1236        ostringstream str;
1237        str << "Adding new run number " << newRun << " that was issued on " << time;
1238        Debug(str);
1239    }
1240    //Add new run number to run number list
1241    fRunNumber.push_back(RunNumberType());
1242    fRunNumber.back().runNumber = uint32_t(newRun);
1243    fRunNumber.back().time = time;
1244
1245    ostringstream str;
1246    str << "The new run number is: " << fRunNumber.back().runNumber;
1247    Message(str);
1248
1249    if (GetCurrentState() != kSM_Logging)
1250        return;
1251    //open the log and report files
1252    OpenRunFile(fRunNumber.back());
1253}
1254// --------------------------------------------------------------------------
1255//
1256//! Checks whether or not the current info is a run number.
1257//! If so, then remember it. A run number is required to open the run-log file
1258//! @param I
1259//!        the current DimInfo
1260//
1261void DataLogger::CheckForRunNumber(DimInfo* I)
1262{
1263    if (strstr(I->getName(), fRunNumberInfo) != NULL)
1264    {//assumes that the run number is an integer
1265        //check if some run number entries can be deleted leave one so that two remain after adding the new one
1266        AddNewRunNumber(I->getLonglong(), Time(I->getTimestamp(), I->getTimestampMillisecs()*1000));
1267    }
1268}
1269
1270// --------------------------------------------------------------------------
1271//
1272//! write infos to log files.
1273//! @param I
1274//!     The current DimInfo
1275//! @param sub
1276//!        The dataLogger's subscription corresponding to this DimInfo
1277//
1278void DataLogger::ReportPlease(DimInfo* I, SubscriptionType& sub)
1279{
1280
1281
1282    //should we log or report this info ? (i.e. is it a message ?)
1283    bool isItaReport = ((strstr(I->getName(), "Message") == NULL) && (strstr(I->getName(), "MESSAGE") == NULL));
1284    if (I->getFormat()[0] == 'C')
1285        isItaReport = false;
1286
1287    if (!fNightlyReportFile.is_open())
1288        return;
1289
1290    //Check whether we should close and reopen daily text files or not
1291    //This should work in any case base of the following:
1292    // - fDailyFileDayChangedAlready is initialized to true. So if the dataLogger is started around noon, no file will be closed
1293    // - fDailyFileDayChangedAlready is set to false if (time != 12), so the file will be closed and reopened only if the logger runs since before noon (which is the required behavior)
1294    //This only applies to text files. Fits are closed and reopened based on the last and current service received time.
1295    //this was not applicable to text files, because as they gather several services, we have no guarantee that the received time will be greater than the previous one,
1296    //which could lead to several close/reopen instead of only one.
1297    if (Time().h() == 12 && !fDailyFileDayChangedAlready)
1298    {
1299        if (fDebugIsOn)
1300            Debug("Its Noon! Closing and reopening daily text files");
1301
1302        fNightlyLogFile.close();
1303        fNightlyReportFile.close();
1304
1305        fFullNightlyLogFileName = CompileFileName(fNightlyFilePath, "", "log");
1306        OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName);
1307
1308        fFullNightlyReportFileName = CompileFileName(fNightlyFilePath, "", "rep");
1309        OpenTextFilePlease(fNightlyReportFile, fFullNightlyReportFileName);
1310
1311        fDailyFileDayChangedAlready = true;
1312    }
1313    if (Time().h() != 12 && fDailyFileDayChangedAlready)
1314        fDailyFileDayChangedAlready = false;
1315
1316    //create the converter for that service
1317    if ((!sub.fConv.get()) && isItaReport)
1318    {
1319        //trick the converter in case of 'C'. why do I do this ? well simple: the converter checks that the right number
1320        //of bytes was written. because I skip 'C' with fits, the bytes will not be allocated, hence the "size copied ckeck"
1321        //of the converter will fail, hence throwing an exception.
1322        string fakeFormat(I->getFormat());
1323        if (fakeFormat[fakeFormat.size()-1] == 'C')
1324            fakeFormat = fakeFormat.substr(0, fakeFormat.size()-1);
1325        sub.fConv = shared_ptr<Converter>(new Converter(Out(), I->getFormat()));
1326        if (!sub.fConv)
1327        {
1328            ostringstream str;
1329            str << "Couldn't properly parse the format... service " << sub.dimInfo->getName() << " ignored.";
1330            Error(str);
1331            return;   
1332        }
1333    }
1334    //construct the header
1335    ostringstream header;
1336    const Time cTime(I->getTimestamp(), I->getTimestampMillisecs()*1000);
1337    fQuality = I->getQuality();
1338    fMjD = cTime.Mjd();
1339
1340    //figure out which run file should be used
1341    ofstream* targetRunFile = NULL;
1342    RunNumberType* cRunNumber = NULL;
1343    if (GetCurrentState() == kSM_Logging)
1344    {
1345        list<RunNumberType>::reverse_iterator rit;
1346        for (rit=fRunNumber.rbegin(); rit!=fRunNumber.rend(); rit++)
1347        {
1348            if (rit->time < cTime) //this is the run number that we want to use
1349            {
1350                //Find something better to convert iterator to pointer than the ugly line below....
1351                cRunNumber = &(*rit);
1352                sub.runNumber = rit->runNumber;
1353#ifdef RUN_LOGS
1354                targetRunFile = isItaReport ? (rit->reportFile).get() : (rit->logFile).get();
1355#else
1356                targetRunFile = isItaReport ? (rit->reportFile).get() : NULL;
1357#endif
1358                break;
1359            }
1360        }
1361        if (rit == fRunNumber.rend() && fRunNumber.size() != 0)
1362        {
1363            ostringstream str;
1364            str << "Could not find an appropriate run number for info coming at time: " << cTime;
1365            Error(str);
1366            Error("Active run numbers: ");
1367            for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
1368            {
1369                str.str("");
1370                str << rit->runNumber;
1371                Error(str);
1372            }
1373
1374        }
1375    }
1376
1377    if (isItaReport)
1378    {
1379        //write text header
1380        header << I->getName() << " " << fQuality << " ";
1381        header << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
1382        header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
1383        header << cTime.ms() << " " << I->getTimestamp() << " ";
1384
1385        string text;
1386        try
1387        {
1388            text = sub.fConv->GetString(I->getData(), I->getSize());
1389        }
1390        catch (const runtime_error &e)
1391        {
1392            Out() << kRed << e.what() << endl;
1393            ostringstream str;
1394            str << "Could not properly parse the data for service " << sub.dimInfo->getName();
1395            str << " reason: " << e.what() << ". Entry ignored";
1396            Error(str);
1397            return;
1398        }
1399
1400        if (text.empty())
1401        {
1402            ostringstream str;
1403            str << "Service " << sub.dimInfo->getName() << " sent an empty string";
1404            Info(str);
1405            return;
1406        }
1407        //replace bizarre characters by white space
1408        replace(text.begin(), text.end(), '\n', '\\');
1409        replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
1410       
1411        //write entry to Nightly report
1412        if (fNightlyReportFile.is_open())
1413        {
1414            fNightlyReportFile << header.str() << text << endl;
1415            CheckForOfstreamError(fNightlyReportFile);
1416        }
1417        //write entry to run-report
1418        if (targetRunFile && targetRunFile->is_open())
1419        {
1420            *targetRunFile << header.str() << text << endl;
1421            CheckForOfstreamError(*targetRunFile);
1422        }
1423    }
1424    else
1425    {//write entry to both Nightly and run logs
1426        string n = I->getName();
1427        ostringstream msg;
1428        msg << n << ": " << I->getString();
1429
1430        if (fNightlyLogFile.is_open())
1431        {
1432            MessageImp nightlyMess(fNightlyLogFile);
1433            nightlyMess.Write(cTime, msg.str().c_str(), fQuality);
1434            CheckForOfstreamError(fNightlyLogFile);
1435        }
1436        if (targetRunFile && targetRunFile->is_open())
1437        {
1438            MessageImp runMess(*targetRunFile);
1439            runMess.Write(cTime, msg.str().c_str(), fQuality);
1440            CheckForOfstreamError(*targetRunFile);
1441        }
1442    }
1443
1444#ifdef HAVE_FITS
1445    if (isItaReport)
1446    {
1447        //check if the last received event was before noon and if current one is after noon.
1448        //if so, close the file so that it gets reopened.
1449        if (sub.nightlyFile.IsOpen())
1450            if ((sub.lastReceivedEvent != Time::None) && (sub.lastReceivedEvent.h() < 12) && (cTime.h() >= 12))
1451            {
1452                sub.nightlyFile.Close();
1453            }
1454        sub.lastReceivedEvent = cTime;
1455        if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen() || sub.runNumber != sub.runFile.fRunNumber)
1456            OpenFITSFilesPlease(sub, cRunNumber);
1457        WriteToFITS(sub);
1458    }   
1459#endif
1460
1461}
1462
1463// --------------------------------------------------------------------------
1464//
1465//! print the dataLogger's current state. invoked by the PRINT command
1466//! @param evt
1467//!        the current event. Not used by the method
1468//! @returns
1469//!        the new state. Which, in that case, is the current state
1470//!
1471int DataLogger::PrintStatePlease(const Event& )
1472{
1473    Message("------------------------------------------");
1474    Message("------- DATA LOGGER CURRENT STATE --------");
1475    Message("------------------------------------------");
1476
1477    //print the path configuration
1478    Message("Nightly Path: " + boost::filesystem::system_complete(boost::filesystem::path(fNightlyFilePath)).directory_string());
1479    Message("Run Path: " + boost::filesystem::system_complete(boost::filesystem::path(fRunFilePath)).directory_string());
1480
1481    //print active run numbers
1482    ostringstream str;
1483    str << "Active Run Numbers:";
1484    for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
1485        str << " " << it->runNumber;
1486    if (fRunNumber.size()==0)
1487        str << " <none>";
1488    Message(str);
1489
1490    //print all the open files.
1491    Message("------------ OPENED FILES ----------------");
1492    if (fNightlyLogFile.is_open())
1493        Message("Nightly log-file: OPEN");
1494
1495    if (fNightlyReportFile.is_open())
1496        Message("Nightly report-file: OPEN");
1497
1498    for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
1499    {
1500#ifdef RUN_LOGS
1501        if (it->logFile->is_open())
1502            Message("Run log-file: " + it->logName + " (OPEN)");
1503#endif
1504        if (it->reportFile->is_open())
1505            Message("Run report-file: " + it->reportName + " (OPEN)");
1506    }
1507
1508    DataLoggerStats statVar;
1509    /*const bool statWarning =*/ calculateTotalSizeWritten(statVar, true);
1510
1511    Message("----------------- STATS ------------------");
1512    str.str("");
1513    str << "Total Size written: " << statVar.sizeWritten << " bytes.";
1514        Message(str);
1515    str.str("");
1516    str << "Disk free space:    " << statVar.freeSpace   << " bytes.";
1517        Message(str);
1518    str.str("");
1519    str << "Statistics are updated every " << fStatsPeriodDuration << " seconds";
1520    if (fStatsPeriodDuration != 0)
1521        Message(str);
1522    else
1523        Message("Statistics updates are currently disabled");
1524
1525    Message("------------ DIM SUBSCRIPTIONS -----------");
1526    str.str("");
1527    str << "There are " << fNumSubAndFitsData.numSubscriptions << " active DIM subscriptions.";
1528    Message(str);
1529    for (map<const string, map<string, SubscriptionType> >::const_iterator it=fServiceSubscriptions.begin(); it!= fServiceSubscriptions.end();it++)
1530    {
1531        Message("Server "+it->first);
1532        for (map<string, SubscriptionType>::const_iterator it2=it->second.begin(); it2!=it->second.end(); it2++)
1533            Message(" -> "+it2->first);
1534    }
1535    Message("--------------- BLOCK LIST ---------------");
1536    for (set<string>::const_iterator it=fBlackList.begin(); it != fBlackList.end(); it++)
1537        Message(" -> "+*it);
1538    if (fBlackList.size()==0)
1539        Message(" <empty>");
1540
1541    Message("--------------- ALLOW LIST ---------------");
1542    for (set<string>::const_iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++)
1543        Message(" -> "+*it);
1544    if (fWhiteList.size()==0)
1545        Message(" <empty>");
1546
1547    Message("-------------- GROUPING LIST -------------");
1548    Message("The following servers and/or services will");
1549    Message("be grouped into a single fits file:");
1550    for (set<string>::const_iterator it=fGrouping.begin(); it != fGrouping.end(); it++)
1551        Message(" -> "+*it);
1552    if (fGrouping.size()==0)
1553        Message(" <no grouping>");
1554
1555    Message("------------------------------------------");
1556    Message("-------- END OF DATA LOGGER STATE --------");
1557    Message("------------------------------------------");
1558
1559    return GetCurrentState();
1560}
1561
1562// --------------------------------------------------------------------------
1563//
1564//! turn debug mode on and off
1565//! @param evt
1566//!        the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1
1567//! @returns
1568//!        the new state. Which, in that case, is the current state
1569//!
1570int DataLogger::SetDebugOnOff(const Event& evt)
1571{
1572    bool backupDebug = fDebugIsOn;
1573    fDebugIsOn = evt.GetBool();
1574    if (fDebugIsOn == backupDebug)
1575        Warn("Warning: debug mode was already in the requested state");
1576    else
1577    {
1578        ostringstream str;
1579        str << "Debug mode is now " << fDebugIsOn;
1580        Message(str);
1581    }
1582    return GetCurrentState();
1583}
1584// --------------------------------------------------------------------------
1585//
1586//! set the statistics update period duration. 0 disables the statistics
1587//! @param evt
1588//!        the current event. contains the new duration.
1589//! @returns
1590//!        the new state. Which, in that case, is the current state
1591//!
1592int DataLogger::SetStatsPeriod(const Event& evt)
1593{
1594    float backupDuration = fStatsPeriodDuration;
1595    fStatsPeriodDuration = evt.GetFloat();
1596    if (fStatsPeriodDuration < 0)
1597    {
1598        Error("Statistics period duration should be greater than zero. Discarding provided value.");
1599        fStatsPeriodDuration = backupDuration;
1600        return GetCurrentState();   
1601    }
1602    if (!finite(fStatsPeriodDuration))// != fStatsPeriodDuration)
1603    {
1604        Error("Provided duration does not appear to be a valid float. discarding it.");
1605        fStatsPeriodDuration = backupDuration;
1606        return GetCurrentState();   
1607    }
1608    if (backupDuration == fStatsPeriodDuration)
1609        Warn("Warning: statistics period was not modified: supplied value already in use");
1610    else
1611    {
1612        if (fStatsPeriodDuration == 0.0f)
1613            Message("Statistics are now OFF");
1614        else
1615        {
1616            ostringstream str;
1617            str << "Statistics period is now " << fStatsPeriodDuration << " seconds";
1618            Message(str);
1619        }   
1620    }
1621    return GetCurrentState();
1622}
1623// --------------------------------------------------------------------------
1624//
1625//! set the opened files service on or off.
1626//! @param evt
1627//!        the current event. contains the instruction string. similar to setdebugonoff
1628//! @returns
1629//!        the new state. Which, in that case, is the current state
1630//!
1631int DataLogger::SetOpenedFilesOnOff(const Event& evt)
1632{
1633    bool backupOpened = fOpenedFilesIsOn;
1634    fOpenedFilesIsOn = evt.GetBool();
1635    if (fOpenedFilesIsOn == backupOpened)
1636        Warn("Warning: opened files service mode was already in the requested state");
1637    else
1638    {
1639        ostringstream str;
1640        str << "Opened files service mode is now " << fOpenedFilesIsOn;
1641        Message(str);
1642    }
1643    return GetCurrentState();
1644   
1645}
1646// --------------------------------------------------------------------------
1647//
1648//! set the number of subscriptions and opened fits on and off
1649//! @param evt
1650//!        the current event. contains the instruction string. similar to setdebugonoff
1651//! @returns
1652//!        the new state. Which, in that case, is the current state
1653//!
1654int DataLogger::SetNumSubsAndFitsOnOff(const Event& evt)
1655{
1656    const bool backupSubs = fNumSubAndFitsIsOn;
1657    fNumSubAndFitsIsOn = evt.GetBool();
1658
1659    if (fNumSubAndFitsIsOn == backupSubs)
1660        Warn("Warning: Number of subscriptions service mode was already in the requested state");
1661    else
1662    {
1663        ostringstream str;
1664        str << "Number of subscriptions service mode is now " << fNumSubAndFitsIsOn;
1665        Message(str);
1666    }
1667    return GetCurrentState();
1668}
1669// --------------------------------------------------------------------------
1670//
1671//!    Sets the path to use for the Nightly log file.
1672//! @param evt
1673//!     the event transporting the path
1674//! @returns
1675//!        currently only the current state.
1676//
1677int DataLogger::ConfigureNightlyFileName(const Event& evt)
1678{
1679    if (evt.GetText() != NULL)
1680    {
1681        string givenPath = string(evt.GetText());
1682        if (!DoesPathExist(givenPath))
1683        {
1684            Error("Provided path is not a valid folder. Ignored");
1685            return GetCurrentState();
1686        }
1687        fNightlyFilePath = givenPath;
1688        Message("New Nightly folder specified: " + fNightlyFilePath);
1689    }
1690    else
1691        Error("Empty Nightly folder given. Please specify a valid path.");
1692
1693    return GetCurrentState();
1694}
1695// --------------------------------------------------------------------------
1696//
1697//! Sets the path to use for the run log file.
1698//! @param evt
1699//!        the event transporting the path
1700//! @returns
1701//!     currently only the current state
1702int DataLogger::ConfigureRunFileName(const Event& evt)
1703{
1704    if (evt.GetText() == NULL)
1705    {
1706        Error("Empty Nightly folder given. Please specify a valid path");
1707        return GetCurrentState();
1708    }
1709
1710    const string givenPath = string(evt.GetText());
1711    if (!DoesPathExist(givenPath))
1712    {
1713        Error("Provided path is not a valid folder. Ignored");
1714        return GetCurrentState();
1715    }
1716
1717    fRunFilePath = givenPath;
1718    Message("New Run folder specified: " + fRunFilePath);
1719
1720    return GetCurrentState();
1721}
1722// --------------------------------------------------------------------------
1723//
1724//! Sets the run number.
1725//! @param evt
1726//!        the event transporting the run number
1727//! @returns
1728//!     currently only the current state
1729int DataLogger::ConfigureRunNumber(const Event& evt)
1730{
1731    AddNewRunNumber(evt.GetXtra(), evt.GetTime());
1732    return GetCurrentState();
1733}
1734// --------------------------------------------------------------------------
1735//
1736//! Notifies the DIM service that a particular file was opened
1737//! @ param name the base name of the opened file, i.e. without path nor extension.
1738//!     WARNING: use string instead of string& because I pass values that do not convert to string&.
1739//!        this is not a problem though because file are not opened so often.
1740//! @ param type the type of the opened file. 0 = none open, 1 = log, 2 = text, 4 = fits
1741inline void DataLogger::NotifyOpenedFile(const string &name, int type, DimDescribedService* service)
1742{
1743    if (!fOpenedFilesIsOn)
1744        return;
1745
1746    if (fDebugIsOn)
1747    {
1748        ostringstream str;
1749        str << "Updating " << service->getName() << " file '" << name << "' (type=" << type << ")";
1750        Debug(str);
1751
1752        str.str("");
1753        str << "Num subs: " << fNumSubAndFitsData.numSubscriptions << " Num open FITS: " << fNumSubAndFitsData.numOpenFits;
1754        Debug(str);
1755    }
1756
1757    if (name.size()+1 > FILENAME_MAX)
1758    {
1759        Error("Provided file name '" + name + "' is longer than allowed file name length");
1760    }
1761
1762    OpenFileToDim fToDim;
1763    fToDim.code = type;
1764    memcpy(fToDim.fileName, name.c_str(), name.size()+1);
1765
1766    service->setData(reinterpret_cast<void*>(&fToDim), name.size()+1+sizeof(int));
1767    service->setQuality(0);
1768    service->updateService();
1769}
1770// --------------------------------------------------------------------------
1771//
1772//! Implements the Start transition.
1773//! Concatenates the given path for the Nightly file and the filename itself (based on the day),
1774//! and tries to open it.
1775//! @returns
1776//!        kSM_NightlyOpen if success, kSM_BadNightlyConfig if failure
1777int DataLogger::StartPlease()
1778{
1779    if (fDebugIsOn)
1780    {
1781        Debug("Starting...");   
1782    }
1783    fFullNightlyLogFileName = CompileFileName(fNightlyFilePath, "", "log");
1784    OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName);
1785
1786    fFullNightlyReportFileName = CompileFileName(fNightlyFilePath, "", "rep");
1787    OpenTextFilePlease(fNightlyReportFile, fFullNightlyReportFileName);
1788
1789    if (!fNightlyLogFile.is_open() || !fNightlyReportFile.is_open())
1790    {   
1791        ostringstream str;
1792        str << "Something went wrong while openning nightly files " << fFullNightlyLogFileName << " and " << fFullNightlyReportFileName;
1793        Error(str);
1794        return kSM_BadNightlyConfig;
1795    }
1796    //get the size of the newly opened file.
1797    fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName);
1798    fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName);
1799    fFileSizesMap.clear();
1800    fBaseSizeRun = 0;
1801    fPreviousSize = 0;
1802
1803    //notify that a new file has been opened.
1804    const string baseFileName = CompileFileName(fNightlyFilePath, "", "");
1805    NotifyOpenedFile(baseFileName, 3, fOpenedNightlyFiles);
1806
1807    fOpenedNightlyFits.clear();
1808   
1809    return kSM_NightlyOpen;     
1810}
1811
1812#ifdef HAVE_FITS
1813// --------------------------------------------------------------------------
1814//
1815//! open if required a the FITS files corresponding to a given subscription
1816//! @param sub
1817//!     the current DimInfo subscription being examined
1818void DataLogger::OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber)
1819{
1820    string serviceName(sub.dimInfo->getName());
1821
1822    //if run number has changed, reopen a new fits file with the correct run number.
1823     if (sub.runFile.IsOpen() && sub.runFile.fRunNumber != sub.runNumber)
1824     {
1825         if (fDebugIsOn)
1826             Debug("Run number changed. Closing " + sub.runFile.fFileName);
1827         sub.runFile.Close();
1828     }
1829
1830     //we must check if we should group this service subscription to a single fits file before we replace the / by _
1831    bool hasGrouping = false;
1832    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
1833    {//will we find this service in the grouping list ?
1834        for (set<string>::const_iterator it=fGrouping.begin(); it!=fGrouping.end(); it++)
1835        {
1836            if (serviceName.find(*it) != string::npos)
1837            {
1838                hasGrouping = true;
1839                break;
1840            }
1841        }
1842    }
1843    hasGrouping = true;
1844    for (unsigned int i=0;i<serviceName.size(); i++)
1845    {
1846        if (serviceName[i] == '/')
1847        {
1848            serviceName[i] = '_';
1849            break;   
1850        }   
1851    }
1852    //we open the NightlyFile anyway, otherwise this function shouldn't have been called.
1853    if (!sub.nightlyFile.IsOpen())
1854    {
1855        string partialName = CompileFileName(fNightlyFilePath, serviceName, "fits");
1856
1857        const string fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1858        if (!sub.fitsBufferAllocated)
1859            AllocateFITSBuffers(sub);
1860        //get the size of the file we're about to open
1861        if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update)
1862            fOpenedNightlyFits[fileNameOnly].push_back(serviceName);
1863
1864        if (!sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, 0))
1865        {
1866            SetCurrentState(kSM_WriteError);
1867            return;
1868        }
1869        //notify the opening
1870        const string baseFileName = CompileFileName(fNightlyFilePath, "", "");
1871        NotifyOpenedFile(baseFileName, 7, fOpenedNightlyFiles);
1872        if (fNumSubAndFitsIsOn)
1873            fNumSubAndFits->updateService();
1874        if (fDebugIsOn)
1875        {
1876            ostringstream str;
1877            str << "Opened Nightly FITS: " << partialName << " and table: " << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
1878            Debug(str);
1879        }
1880    }
1881    //do the actual file open
1882    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging) && sub.runNumber != 0)
1883    {//buffer for the run file have already been allocated when doing the Nightly file
1884        string fileNameOnly;
1885        string partialName;
1886        if (hasGrouping)
1887        {
1888            partialName = CompileFileName(fRunFilePath, sub.runNumber, "group", "fits");
1889            fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1890        }
1891        else
1892        {
1893            partialName = CompileFileName(fRunFilePath, sub.runNumber, serviceName, "fits");
1894            fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1895        }
1896        //get the size of the file we're about to open
1897        if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update)
1898            cRunNumber->openedFits[fileNameOnly].push_back(serviceName);
1899        else
1900            if (hasGrouping)
1901            {
1902             cRunNumber->addServiceToOpenedFits(fileNameOnly, serviceName);
1903            }
1904
1905        if (hasGrouping && (!cRunNumber->runFitsFile.get()))
1906            try
1907            {
1908                cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>(new CCfits::FITS(partialName, CCfits::RWmode::Write));
1909                (fNumSubAndFitsData.numOpenFits)++;
1910            }   
1911            catch (CCfits::FitsException e)
1912            {
1913                ostringstream str;
1914                str << "Could not open FITS Run file " << partialName << " reason: " << e.message();
1915                Error(str);
1916                cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>();//NULL;
1917            }
1918
1919        const string baseFileName = CompileFileName(fRunFilePath, sub.runNumber, "", "");
1920        NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles);// + '_' + serviceName, 4);
1921        if (hasGrouping)
1922        {
1923            if (!sub.runFile.Open(partialName, serviceName, (cRunNumber->runFitsFile).get(), &fNumSubAndFitsData.numOpenFits, this, sub.runNumber))
1924            {
1925                SetCurrentState(kSM_WriteError);
1926                return;
1927            }
1928        }
1929        else
1930        {
1931            if (sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber))
1932            {
1933                SetCurrentState(kSM_WriteError);
1934                return;
1935            }
1936        }
1937       if (fNumSubAndFitsIsOn)
1938           fNumSubAndFits->updateService();
1939           if (fDebugIsOn)
1940        {
1941            ostringstream str;
1942            str << "Opened Run FITS: " << partialName << " and table: " << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
1943            Debug(str);
1944        }
1945    }
1946}   
1947// --------------------------------------------------------------------------
1948//
1949//! Allocates the required memory for a given pair of fits files (nightly and run)
1950//! @param sub the subscription of interest.
1951//
1952void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
1953{
1954    //Init the time columns of the file
1955    Description dateDesc(string("Time"), string("Modified Julian Date"), string("MjD"));
1956    sub.nightlyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1957    sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1958
1959    Description QoSDesc("Qos", "Quality of service", "None");
1960    sub.nightlyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1961    sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1962
1963    const Converter::FormatList flist = sub.fConv->GetList();
1964    // Compilation failed
1965    if (!sub.fConv->valid())
1966    {
1967        Error("Compilation of format string failed.");
1968        return;
1969    }
1970
1971    //we've got a nice structure describing the format of this service's messages.
1972    //Let's create the appropriate FITS columns
1973    int size = sub.dimInfo->getSize();
1974
1975    vector<string> dataFormatsLocal;
1976    for (unsigned int i=0;i<flist.size()-1;i++)
1977    {
1978         ostringstream dataQualifier;
1979
1980         dataQualifier << flist[i].second.first;
1981         switch (flist[i].first.first->name()[0])
1982         {
1983             case 'c':
1984             case 'C':
1985                 dataQualifier.str("S");
1986             break;
1987             case 's':
1988                 dataQualifier << "I";
1989             break;
1990             case 'i':
1991             case 'I':
1992                 dataQualifier << "J";
1993             break;
1994             case 'l':
1995             case 'L':
1996                 dataQualifier << "J";
1997             break;
1998             case 'f':
1999             case 'F':
2000                 dataQualifier << "E";
2001             break;
2002             case 'd':
2003             case 'D':
2004                 dataQualifier << "D";
2005             break;
2006             case 'x':
2007             case 'X':
2008                 dataQualifier << "K";
2009             break;
2010             case 'S':
2011                 size--;
2012                 //for strings, the number of elements I get is wrong. Correct it
2013                 dataQualifier.str(""); //clear
2014                 dataQualifier << size <<  "A";
2015             break;
2016             
2017             default:
2018                 Fatal("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 1198.");
2019         };
2020         //we skip the variable length strings for now (in fits only)
2021         if (dataQualifier.str() != "S")
2022             dataFormatsLocal.push_back(dataQualifier.str());
2023     }
2024     sub.nightlyFile.InitDataColumns(GetDescription(sub.server, sub.service), dataFormatsLocal, sub.dimInfo->getData(), size);
2025     sub.runFile.InitDataColumns(GetDescription(sub.server, sub.service), dataFormatsLocal, sub.dimInfo->getData(), size);
2026     sub.fitsBufferAllocated = true;
2027}
2028// --------------------------------------------------------------------------
2029//
2030//! write a dimInfo data to its corresponding FITS files
2031//
2032void DataLogger::WriteToFITS(SubscriptionType& sub)
2033{
2034        //nightly File status (open or not) already checked
2035        if (sub.nightlyFile.IsOpen())
2036        {
2037            if (!sub.nightlyFile.Write(sub.fConv.get()))
2038                SetCurrentState(kSM_WriteError);
2039         }
2040
2041        if (sub.runFile.IsOpen())
2042        {
2043            if (!sub.runFile.Write(sub.fConv.get()))
2044                SetCurrentState(kSM_WriteError);
2045        }
2046}
2047#endif //if has_fits
2048
2049std::string DataLogger::SetCurrentState(int state, const char *txt, const std::string &cmd)
2050{
2051    if (state == kSM_WriteError && GetCurrentState() == kSM_WriteError)
2052        return "";
2053    return StateMachineImp::SetCurrentState(state, txt, cmd);
2054}
2055// --------------------------------------------------------------------------
2056//
2057//! Implements the StartRun transition.
2058//! Concatenates the given path for the run file and the filename itself (based on the run number),
2059//! and tries to open it.
2060//! @returns
2061//!        kSM_Logging if success, kSM_BadRunConfig if failure.
2062int DataLogger::StartRunPlease()
2063{
2064    if (fDebugIsOn)
2065    {
2066        Debug("Starting Run Logging...");   
2067    }
2068    //open all the relevant run-files. i.e. all the files associated with run numbers.
2069    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
2070        OpenRunFile(*it);
2071
2072    return kSM_Logging;
2073}
2074
2075#ifdef HAVE_FITS
2076// --------------------------------------------------------------------------
2077//
2078//! Create a fits group file with all the run-fits that were written (either daily or run)
2079//! @param filesToGroup a map of filenames mapping to table names to be grouped (i.e. a
2080//!        single file can contain several tables to group
2081//! @param runNumber the run number that should be used for grouping. 0 means nightly group
2082//
2083void DataLogger::CreateFitsGrouping(map<string, vector<string> > & filesToGroup, int runNumber)
2084{
2085    if (fDebugIsOn)
2086    {
2087        ostringstream str;
2088        str << "Creating fits group for ";
2089        if (runNumber != 0)
2090            str << "run files";
2091        else
2092            str << "nightly files";
2093        Debug(str);
2094    }
2095    //create the FITS group corresponding to the ending run.
2096    CCfits::FITS* groupFile;
2097    unsigned int numFilesToGroup = 0;
2098    for (map<string, vector<string> >::const_iterator it=filesToGroup.begin(); it != filesToGroup.end(); it++)
2099    {
2100        numFilesToGroup += it->second.size();
2101    }
2102    if (fDebugIsOn)
2103    {
2104        ostringstream str;
2105        str << "There are " << numFilesToGroup << " tables to group";
2106        Debug(str);
2107    }
2108    if (numFilesToGroup <= 1)
2109    {
2110        filesToGroup.clear();
2111        return;
2112    }
2113    string groupName;
2114    if (runNumber != 0)
2115        groupName = CompileFileName(fRunFilePath, runNumber, "", "fits");
2116    else
2117        groupName = CompileFileName(fNightlyFilePath, "", "fits");
2118
2119    CCfits::Table* groupTable;
2120    int maxCharLength = 50;//FILENAME_MAX;
2121    try
2122    {
2123        groupFile = new CCfits::FITS(groupName, CCfits::RWmode::Write);
2124        //setup the column names
2125        ostringstream pathTypeName;
2126        pathTypeName << maxCharLength << "A";
2127        vector<string> names;
2128        vector<string> dataTypes;
2129        names.push_back("MEMBER_XTENSION");
2130        dataTypes.push_back("8A");
2131        names.push_back("MEMBER_URI_TYPE");
2132        dataTypes.push_back("3A");
2133        names.push_back("MEMBER_LOCATION");
2134        dataTypes.push_back(pathTypeName.str());
2135        names.push_back("MEMBER_NAME");
2136        dataTypes.push_back(pathTypeName.str());
2137
2138        groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
2139//TODO handle the case when the logger was stopped and restarted during the same day, i.e. the grouping file must be updated
2140     }
2141     catch (CCfits::FitsException e)
2142     {
2143         ostringstream str;
2144         str << "Could not open or create FITS table GROUPING in  file " << groupName << " reason: " << e.message();
2145         Error(str);
2146         return;
2147     }
2148
2149    //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about.
2150    //use cfitsio routines instead
2151    groupTable->makeThisCurrent();
2152    //create appropriate buffer.
2153    const unsigned int n = 8 + 3 + 2*maxCharLength + 1; //+1 for trailling character
2154
2155    vector<unsigned char> realBuffer;
2156    realBuffer.resize(n);
2157    unsigned char* fitsBuffer = &realBuffer[0];
2158    memset(fitsBuffer, 0, n);
2159
2160    char* startOfExtension = reinterpret_cast<char*>(fitsBuffer);
2161    char* startOfURI       = reinterpret_cast<char*>(&fitsBuffer[8]);
2162    char* startOfLocation  = reinterpret_cast<char*>(&fitsBuffer[8 + 3]);
2163    char* startOfName      = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]);
2164
2165    strcpy(startOfExtension, "BINTABLE");
2166    strcpy(startOfURI,       "URL");
2167
2168    int i=1;
2169    for (map<string, vector<string> >::const_iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++)
2170        for (vector<string>::const_iterator jt=it->second.begin(); jt != it->second.end(); jt++, i++)
2171        {
2172            strcpy(startOfLocation, it->first.c_str());
2173            strcpy(startOfName, jt->c_str());
2174
2175            if (fDebugIsOn)
2176            {
2177                ostringstream str;
2178                str << "Grouping " << it->first << " " << *jt;
2179                Debug(str);
2180            }
2181
2182            int status = 0;
2183            fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status);
2184            if (status)
2185            {
2186                ostringstream str;
2187                str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status;
2188                Error(str);
2189            }
2190        }
2191
2192    filesToGroup.clear();
2193    delete groupFile;
2194}
2195#endif //HAVE_FITS
2196
2197// --------------------------------------------------------------------------
2198//
2199//! Implements the StopRun transition.
2200//! Attempts to close the run file.
2201//! @returns
2202//!        kSM_WaitingRun if success, kSM_FatalError otherwise
2203int DataLogger::StopRunPlease()
2204{
2205
2206    if (fDebugIsOn)
2207    {
2208        Debug("Stopping Run Logging...");   
2209    }
2210    for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
2211    {
2212#ifdef RUN_LOGS
2213        if (!it->logFile->is_open() || !it->reportFile->is_open())
2214#else
2215        if (!it->reportFile->is_open())
2216#endif
2217            return kSM_FatalError;
2218#ifdef RUN_LOGS
2219        it->logFile->close();
2220#endif
2221        it->reportFile->close();
2222    }
2223
2224#ifdef HAVE_FITS
2225    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2226        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2227        {
2228            if (j->second.runFile.IsOpen())
2229                j->second.runFile.Close();
2230        }
2231#endif
2232    NotifyOpenedFile("", 0, fOpenedRunFiles);
2233    if (fNumSubAndFitsIsOn)
2234        fNumSubAndFits->updateService();
2235
2236    while (fRunNumber.size() > 0)
2237    {
2238        RemoveOldestRunNumber();
2239    }
2240
2241    return kSM_WaitingRun;
2242}
2243// --------------------------------------------------------------------------
2244//
2245//! Implements the Stop and Reset transitions.
2246//! Attempts to close any openned file.
2247//! @returns
2248//!     kSM_Ready
2249int DataLogger::GoToReadyPlease()
2250{
2251   if (fDebugIsOn)
2252   {
2253        Debug("Going to the Ready state...");
2254   }
2255   if (GetCurrentState() == kSM_Logging)
2256       StopRunPlease();
2257
2258    if (fNightlyLogFile.is_open())
2259        fNightlyLogFile.close();
2260    if (fNightlyReportFile.is_open())
2261        fNightlyReportFile.close();
2262       
2263#ifdef HAVE_FITS
2264    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2265        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2266        {
2267            if (j->second.nightlyFile.IsOpen())
2268                j->second.nightlyFile.Close();;
2269        }
2270#endif
2271    if (GetCurrentState() == kSM_Logging || 
2272        GetCurrentState() == kSM_WaitingRun || 
2273        GetCurrentState() == kSM_NightlyOpen)
2274    { 
2275        NotifyOpenedFile("", 0, fOpenedNightlyFiles);
2276        if (fNumSubAndFitsIsOn)
2277            fNumSubAndFits->updateService();
2278    }
2279#ifdef HAVE_FITS
2280    CreateFitsGrouping(fOpenedNightlyFits, 0);
2281#endif
2282    return kSM_Ready;
2283}
2284// --------------------------------------------------------------------------
2285//
2286//! Implements the transition towards kSM_WaitingRun
2287//! Does nothing really.
2288//!    @returns
2289//!        kSM_WaitingRun
2290int DataLogger::NightlyToWaitRunPlease()
2291{
2292    if (fDebugIsOn)
2293    {
2294        Debug("Going to Wait Run Number state...");   
2295    }
2296    return kSM_WaitingRun;   
2297}
2298// --------------------------------------------------------------------------
2299//
2300//! Setup Logger's configuration from a Configuration object
2301//! @param conf the configuration object that should be used
2302//!
2303bool DataLogger::SetConfiguration(Configuration& conf)
2304{
2305    fDebugIsOn = conf.Get<bool>("debug");
2306
2307    //Set the block or allow list
2308    fBlackList.clear();
2309    fWhiteList.clear();
2310
2311    //Adding entries that should ALWAYS be ignored
2312    //fBlackList.insert("DATA_LOGGER/");
2313    fBlackList.insert("/SERVICE_LIST");
2314    fBlackList.insert("DIS_DNS/");
2315
2316    if (conf.Has("block"))
2317    {
2318        const vector<string> vec = conf.Get<vector<string>>("block");
2319
2320        fBlackList.insert(vec.begin(), vec.end());
2321    }
2322
2323    if (conf.Has("allow"))
2324    {
2325        const vector<string> vec = conf.Get<vector<string>>("allow");
2326        fWhiteList.insert(vec.begin(), vec.end());
2327    }
2328
2329    //Set the grouping
2330    if (conf.Has("group"))
2331    {
2332        const vector<string> vec = conf.Get<vector<string>>("group");
2333        fGrouping.insert(vec.begin(), vec.end());
2334    }
2335    return true;
2336}
2337
2338// --------------------------------------------------------------------------
2339int RunDim(Configuration &conf)
2340{
2341    WindowLog wout;
2342
2343    //log.SetWindow(stdscr);
2344    if (conf.Has("log"))
2345        if (!wout.OpenLogFile(conf.Get<string>("log")))
2346            wout << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;
2347
2348    // Start io_service.Run to use the StateMachineImp::Run() loop
2349    // Start io_service.run to only use the commandHandler command detaching
2350    DataLogger logger(wout);
2351    if (!logger.SetConfiguration(conf))
2352        return -1;
2353
2354    logger.Run(true);
2355
2356    return 0;
2357}
2358// --------------------------------------------------------------------------
2359void RunThread(DataLogger* logger)
2360{
2361    // This is necessary so that the StateMachine Thread can signal the
2362    // Readline thread to exit
2363    logger->Run(true);
2364    Readline::Stop();   
2365}
2366// --------------------------------------------------------------------------
2367template<class T>
2368int RunShell(Configuration &conf)
2369{
2370    static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);
2371
2372    WindowLog &win  = shell.GetStreamIn();
2373    WindowLog &wout = shell.GetStreamOut();
2374
2375    if (conf.Has("log"))
2376        if (!wout.OpenLogFile(conf.Get<string>("log")))
2377            win << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;
2378
2379    DataLogger logger(wout);
2380
2381    if (!logger.SetConfiguration(conf))
2382        return -1;
2383   
2384    shell.SetReceiver(logger);
2385
2386    boost::thread t(boost::bind(RunThread, &logger));
2387   
2388    shell.Run(); // Run the shell
2389   
2390    logger.Stop();
2391   
2392    //Wait until the StateMachine has finished its thread
2393    //before returning and destroyinng the dim objects which might
2394    //still be in use.
2395    t.join();
2396
2397    return 0;
2398}
2399
2400/*
2401 Extract usage clause(s) [if any] for SYNOPSIS.
2402 Translators: "Usage" and "or" here are patterns (regular expressions) which
2403 are used to match the usage synopsis in program output.  An example from cp
2404 (GNU coreutils) which contains both strings:
2405  Usage: cp [OPTION]... [-T] SOURCE DEST
2406    or:  cp [OPTION]... SOURCE... DIRECTORY
2407    or:  cp [OPTION]... -t DIRECTORY SOURCE...
2408 */
2409void PrintUsage()
2410{
2411    cout << "\n"
2412        "The data logger connects to all available Dim services and "
2413        "writes them to ascii and fits files.\n"
2414        "\n"
2415        "The default is that the program is started without user interaction. "
2416        "All actions are supposed to arrive as DimCommands. Using the -c "
2417        "option, a local shell can be initialized. With h or help a short "
2418        "help message about the usage can be brought to the screen.\n"
2419        "\n"
2420        "Usage: dataLogger [-c type] [OPTIONS]\n"
2421        "  or:  dataLogger [OPTIONS]\n";
2422    cout << endl;
2423
2424}
2425// --------------------------------------------------------------------------
2426void PrintHelp()
2427{
2428    /* Additional help text which is printed after the configuration
2429     options goes here */
2430    cout <<
2431        "\n"
2432        "If the allow list has any element, only the servers and/or services "
2433        "specified in the list will be used for subscription. The black list "
2434        "will disable service subscription and has higher priority than the "
2435        "allow list. If the allow list is not present by default all services "
2436        "will be subscribed."
2437        "\n"
2438        "For example, block=DIS_DNS/ will skip all the services offered by "
2439        "the DIS_DNS server, while block=/SERVICE_LIST will skip all the "
2440        "SERVICE_LIST services offered by any server and DIS_DNS/SERVICE_LIST "
2441        "will skip DIS_DNS/SERVICE_LIST.\n"
2442        "\n"
2443        "The commands offered by the dataLoger are the following: \n";
2444    cout << setw(20) << DataLogger::fConfigDay << " : specify the path where to put the nightly files\n";
2445    cout << setw(20) << DataLogger::fConfigRun << " : specify the path where to put the run files\n";
2446    cout << setw(20) << DataLogger::fConfigRunNumber << " : specify the run number\n";
2447    cout << setw(20) << DataLogger::fConfigLog << " : log a particular message\n";
2448    cout << setw(20) << DataLogger::fTransStart << " : start the nightly logging\n";
2449    cout << setw(20) << DataLogger::fTransStop << " : stop the nightly logging\n";
2450    cout << setw(20) << DataLogger::fTransStartRun << " : start the run logging\n";
2451    cout << setw(20) << DataLogger::fTransStopRun << " : stop the run logging\n";
2452    cout << setw(20) << DataLogger::fTransReset << " : stop any logging and/or recover from an error state\n";
2453    cout << setw(20) << DataLogger::fTransWait << " : go to the wait for run number state\n";
2454    cout << setw(20) << DataLogger::fPrintCommand << " : print the current state of the logger to the shell\n";
2455    cout << setw(20) << DataLogger::fDebugOnOff << " : turn on or off the debug mode\n";
2456    cout << setw(20) << DataLogger::fStatsPeriod << " : set the periodicity of the statistics. 0 disable them\n";
2457    cout << endl;
2458}
2459// --------------------------------------------------------------------------
2460void SetupConfiguration(Configuration &conf)
2461{
2462    const string n = conf.GetName()+".log";
2463
2464    po::options_description configp("Program options");
2465    configp.add_options()
2466        ("dns",       var<string>("localhost"),  "Dim nameserver host name (Overwites DIM_DNS_NODE environment variable)")
2467        ("log,l",     var<string>(n), "Write log-file")
2468        ("console,c", var<int>(),     "Use console (0=shell, 1=simple buffered, X=simple unbuffered)")
2469        ;
2470
2471    po::options_description configs("DataLogger options");
2472    configs.add_options()
2473        ("block,b", vars<string>(), "Black-list of services")
2474        ("allow,a", vars<string>(), "White-list of services")
2475        ("debug",   po_bool(),      "Debug mode. Print clear text of received service reports to log-stream")
2476        ("group,g", vars<string>(), "Grouping of services into a single run-Fits")
2477        ;
2478
2479    conf.AddEnv("dns", "DIM_DNS_NODE");
2480
2481    conf.AddOptions(configp);
2482    conf.AddOptions(configs);
2483}
2484// --------------------------------------------------------------------------
2485int main(int argc, const char* argv[])
2486{
2487    Configuration conf(argv[0]);
2488    conf.SetPrintUsage(PrintUsage);
2489    SetupConfiguration(conf);
2490
2491    po::variables_map vm;
2492    try
2493    {
2494        vm = conf.Parse(argc, argv);
2495    }
2496#if BOOST_VERSION > 104000
2497    catch (po::multiple_occurrences &e)
2498    {
2499        cerr << "Program options invalid due to: " << e.what() << " of '" << e.get_option_name() << "'." << endl;
2500        return -1;
2501    }
2502#endif
2503    catch (exception& e)
2504    {
2505        cerr << "Program options invalid due to: " << e.what() << endl;
2506        return -1;
2507    }
2508
2509    if (conf.HasVersion() || conf.HasPrint())
2510        return -1;
2511
2512    if (conf.HasHelp())
2513    {
2514        PrintHelp();
2515        return -1;
2516    }
2517
2518    Dim::Setup(conf.Get<string>("dns"));
2519
2520//    try
2521    {
2522        // No console access at all
2523        if (!conf.Has("console"))
2524            return RunDim(conf);
2525
2526        // Console access w/ and w/o Dim
2527        if (conf.Get<int>("console")==0)
2528            return RunShell<LocalShell>(conf);
2529        else
2530            return RunShell<LocalConsole>(conf);
2531    }
2532/*    catch (exception& e)
2533    {
2534        cerr << "Exception: " << e.what() << endl;
2535        return -1;
2536    }*/
2537
2538    return 0;
2539}
Note: See TracBrowser for help on using the repository browser.