source: trunk/FACT++/src/dataLogger.cc @ 10955

Last change on this file since 10955 was 10955, checked in by lyard, 9 years ago
added WriteError state
File size: 90.5 KB
Line 
1//****************************************************************
2/** @class DataLogger
3 
4  @brief Logs all message and infos between the services
5 
6  This is the main logging class facility.
7  It derives from StateMachineDim and DimInfoHandler. the first parent is here to enforce
8  a state machine behaviour, while the second one is meant to make the dataLogger receive
9  dim services to which it subscribed from.
10  The possible states and transitions of the machine are:
11  \dot
12  digraph datalogger {
13          node [shape=record, fontname=Helvetica, fontsize=10];
14      e [label="Error" color="red"];
15   r [label="Ready"]
16   d [label="NightlyOpen"]
17   w [label="WaitingRun"]
18      l [label="Logging"]
19   b [label="BadNightlyconfig" color="red"]
20   c [label="BadRunConfig" color="red"]
21 
22  e -> r
23  r -> e
24  r -> d
25  r -> b
26  d -> w
27  d -> r
28  w -> r
29  l -> r
30  l -> w
31  b -> d
32  w -> c
33  w -> l
34  b -> r
35  c -> r
36  c -> l
37  }
38  \enddot
39 */
40 //****************************************************************
41#include "Dim.h"
42#include "Event.h"
43#include "Time.h"
44#include "StateMachineDim.h"
45#include "WindowLog.h"
46#include "Configuration.h"
47#include "ServiceList.h"
48#include "Converter.h"
49#include "MessageImp.h"
50#include "LocalControl.h"
51#include "DimDescriptionService.h"
52
53#include "Description.h"
54
55//#include "DimServiceInfoList.h"
56#include "DimNetwork.h"
57//for getting stat of opened files
58#include <unistd.h>
59//for getting disk free space
60#include <sys/statvfs.h>
61//for getting files sizes
62#include <sys/stat.h>
63
64#include <fstream>
65#include <mutex>
66
67#include <boost/bind.hpp>
68#if BOOST_VERSION < 104400
69#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
70#undef BOOST_HAS_RVALUE_REFS
71#endif
72#endif
73#include <boost/thread.hpp>
74
75#ifdef HAVE_FITS
76#include "Fits.h"
77#endif
78
79//Dim structures
80///Distributes the writing statistics
81struct DataLoggerStats {
82    long sizeWritten;
83    long freeSpace;
84    long writingRate;
85};
86///distributes the number of opened subscriptions and fits files
87struct NumSubAndFitsType {
88    int numSubscriptions;
89    int numOpenFits;
90};
91///distributes which files were opened.
92struct OpenFileToDim {
93    int code;
94    char fileName[FILENAME_MAX];
95};
96
97///Run number record. Used to keep track of which run numbers are still active
98struct RunNumberType {
99#ifdef RUN_LOGS
100    ///the run number log file
101    shared_ptr<ofstream> logFile;
102#endif
103    ///the run number report file
104    shared_ptr<ofstream> reportFile;
105#ifdef HAVE_FITS
106    ///the run number group fits file
107    shared_ptr<CCfits::FITS> runFitsFile;
108#endif
109#ifdef RUN_LOGS
110    ///the log filename
111    string logName;
112#endif
113    ///the report filename
114    string reportName;
115    ///the actual run number
116    uint32_t runNumber;
117    ///the time at which the run number was received
118    Time time;
119    ///list of opened fits used to create the fits grouping when the run ends
120    map<string, vector<string> > openedFits;
121    ///default constructor
122    RunNumberType()
123    {
124#ifdef RUN_LOGS
125        logFile = shared_ptr<ofstream>(new ofstream());
126#endif
127        reportFile = shared_ptr<ofstream>(new ofstream());
128#ifdef HAVE_FITS
129        runFitsFile = shared_ptr<CCfits::FITS>();
130#endif
131        runNumber = 0;
132    }
133    ///default destructor
134    ~RunNumberType()
135    {
136
137    }
138
139    void addServiceToOpenedFits(const string& fileName, const string& serviceName)
140    {
141         //most likely I should add this service name.
142         //the only case for which I should not add it is if a service disapeared, hence the file was closed
143         //and reopened again. Unlikely to happen, but well it may
144
145         if (find(openedFits[fileName].begin(), openedFits[fileName].end(),
146                  serviceName)==openedFits[fileName].end())
147            openedFits[fileName].push_back(serviceName);
148    }
149};
150///Dim subscription type. Stores all the relevant info to handle a Dim subscription
151struct SubscriptionType
152{
153#ifdef HAVE_FITS
154    ///Nightly FITS output file
155    Fits    nightlyFile;
156    ///run-specific FITS output file
157    Fits    runFile;
158#endif
159    ///the actual dimInfo pointer
160    shared_ptr<DimStampedInfo> dimInfo;
161    ///the server
162    string server;
163    ///the service
164    string service;
165    ///the converter for outputting the data according to the format
166    shared_ptr<Converter> fConv;
167    ///the current run number used by this subscription
168    uint32_t runNumber;
169    ///time of the latest received event
170    Time lastReceivedEvent;
171    ///whether or not the fits buffer was allocated already
172    bool fitsBufferAllocated;
173
174    ///Dim info constructor
175    SubscriptionType(DimStampedInfo* info=NULL)
176    {
177        dimInfo = shared_ptr<DimStampedInfo>(info);
178        fConv = shared_ptr<Converter>();
179        runNumber = 0;
180        lastReceivedEvent = Time::None;
181        fitsBufferAllocated = false;
182    }
183
184    ///default destructor
185    ~SubscriptionType()
186    {
187    }
188};
189
190class DataLogger : public StateMachineDim, DimServiceInfoList
191{
192public:
193    /// The list of existing states specific to the DataLogger
194    enum
195    {
196        kSM_NightlyOpen = 20, ///< Nightly file openned and writing
197        kSM_WaitingRun = 30, ///< waiting for the run number to open the run file
198        kSM_Logging = 40, ///< both files openned and writing
199        kSM_BadNightlyConfig = 0x101, ///< the folder specified for Nightly logging does not exist or has bad permissions
200        kSM_BadRunConfig = 0x102, ///<  the folder specified for the run logging does not exist or has wrong permissions or no run number
201        kSM_WriteError = 0x103, ///< Denotes that an error occured while writing a file (text or fits).
202    } localstates_t;
203   
204    DataLogger(ostream &out);
205    ~DataLogger(); 
206
207    bool SetConfiguration(Configuration& conf);
208
209private:
210    /************************************************
211     * MEMBER VARIABLES
212     ************************************************/
213    /// ofstream for the NightlyLogfile
214    ofstream fNightlyLogFile;
215    /// ofstream for the Nightly report file
216    ofstream fNightlyReportFile;
217    /// base path of the Nightlyfile
218    string fNightlyFilePath;
219    ///base path of the run file
220    string fRunFilePath;
221    ///run numbers
222    list<RunNumberType> fRunNumber;
223    ///previous run number. to check if changed while logging
224    int fPreviousRunNumber;
225    ///Current Service Quality
226    int fQuality;
227    ///Modified Julian Date
228    double fMjD;
229    ///for obtaining the name of the existing services
230//    ServiceList fServiceList;
231    typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
232    ///All the services to which we have subscribed to, sorted by server name.
233    SubscriptionsListType fServiceSubscriptions;
234    ///full name of the nightly log file
235    string fFullNightlyLogFileName;
236    ///full name of the nightly report file
237    string fFullNightlyReportFileName;
238    ///variables for computing statistics
239    DataLoggerStats fStatVar;
240    ///variable to track when the statistic were last calculated
241    double fPreviousStatsUpdateTime;
242    double fPreviousOldRunNumberCheck;
243    ///mutex to make sure that the Stats are not accessed while updating
244    mutex fStatsMutex;
245    ///boolean to know whether we should close and reopen daily files or not
246    bool fDailyFileDayChangedAlready;
247public:
248    /***************************************************
249     * STATIC COMMAND NAMES
250     ***************************************************/
251    ///Define all the static names
252    static const char* fConfigDay;
253    static const char* fConfigRun;
254    static const char* fConfigRunNumber;
255    static const char* fConfigLog;
256    static const char* fTransStart;
257    static const char* fTransStop;
258    static const char* fTransStartRun;
259    static const char* fTransStopRun;
260    static const char* fTransReset;
261    static const char* fTransWait;
262    static const char* fRunNumberInfo; ///< This is the name of the dimInfo received to specify the run number. It must be updated once the final name will be defined
263    static const char* fPrintCommand;
264    static const char* fDebugOnOff;
265    static const char* fStatsPeriod;
266    static const char* fStartStopOpenedFiles;
267    static const char* fStartStopNumSubsAndFits;
268private:
269    /***************************************************
270     * DIM INFO HANDLER
271     ***************************************************/
272    //overloading of DIM's infoHandler function
273    void infoHandler(); 
274
275    /***************************************************
276     * TRANSITION FUNCTIONS
277     ***************************************************/
278    ///Reporting method for the services info received
279    void ReportPlease(DimInfo* I, SubscriptionType& sub); 
280    ///Configuration of the nightly file path
281    int ConfigureNightlyFileName(const Event& evt); 
282    ///Configuration fo the file name
283    int ConfigureRunFileName(const Event& evt); 
284    ///DEPREC - configuration of the run number
285    int ConfigureRunNumber(const Event& evt); 
286    ///print the current state of the dataLogger
287    int PrintStatePlease(const Event& evt);
288    ///checks whether or not the current info being treated is a run number
289    void CheckForRunNumber(DimInfo* I);
290    /// start transition
291    int StartPlease(); 
292    ///from waiting to logging transition
293    int StartRunPlease(); 
294    /// from logging to waiting transition
295    int StopRunPlease(); 
296    ///stop and reset transition
297    int GoToReadyPlease(); 
298    ///from NightlyOpen to waiting transition
299    int NightlyToWaitRunPlease(); 
300    /// from writing to error
301    std::string SetCurrentState(int state, const char *txt="", const std::string &cmd="");
302#ifdef HAVE_FITS
303    ///Open fits files
304    void OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber);
305    ///Write data to FITS files
306    void WriteToFITS(SubscriptionType& sub);
307    ///Allocate the buffers required for fits
308    void AllocateFITSBuffers(SubscriptionType& sub);
309#endif//has_fits
310
311    /***************************************
312     * DIM SERVICES PROVIDED BY THE DATA LOGGER
313     ***************************************/
314    ///monitoring notification loop
315    void ServicesMonitoring();
316    inline void NotifyOpenedFile(const string &name, int type, DimDescribedService* service);
317    ///services notification thread
318    boost::thread fMonitoringThread;
319    ///end of the monitoring
320    bool fContinueMonitoring;
321    ///stores the size of each file that is or was open
322    map<string, long> fFileSizesMap;
323    ///total size of the opened files BEFORE they were opened by the logger
324    long fBaseSizeNightly;
325    long fPreviousSize;
326    long fBaseSizeRun;
327    ///Service for opened files
328    DimDescribedService* fOpenedNightlyFiles;
329    DimDescribedService* fOpenedRunFiles;
330    DimDescribedService* fNumSubAndFits;
331    NumSubAndFitsType fNumSubAndFitsData;
332    ///Small function for calculating the total size written so far
333    bool calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting);
334
335    /***************************************************
336     * DATA LOGGER's CONFIGURATION STUFF
337     ***************************************************/
338    ///black/white listing
339    set<string> fBlackList;
340    set<string> fWhiteList;
341    ///list of services to be grouped
342    set<string> fGrouping;
343    ///configuration flags
344    bool fDebugIsOn;
345    float fStatsPeriodDuration;
346    bool fOpenedFilesIsOn;
347    bool fNumSubAndFitsIsOn;
348    //functions for controlling the services behavior
349    int SetDebugOnOff(const Event& evt);
350    int SetStatsPeriod(const Event& evt);
351    int SetOpenedFilesOnOff(const Event& evt);
352    int SetNumSubsAndFitsOnOff(const Event& evt);
353
354    ///boolean to prevent DIM update while desctructing the dataLogger
355    bool fDestructing;   
356    /***************************************************
357     * UTILITIES
358     ***************************************************/
359    ///vectors to keep track of opened Fits files, for grouping purposes.
360    map<string, vector<string> > fOpenedNightlyFits;
361    ///creates a group fits file based on a list of files to be grouped
362    void CreateFitsGrouping(map<string, vector<string> >& filesToGroup, int runNumber);
363    ///Open the relevant text files related to a particular run
364    int OpenRunFile(RunNumberType& run);
365    ///add a new run number
366    void AddNewRunNumber(int64_t newRun, Time time);
367    ///removes the oldest run number, and close the relevant files.
368    void RemoveOldestRunNumber();
369    ///retrieves the size of a file
370    off_t GetFileSize(const string&);
371    ///Get the digits of year, month and day for filenames and paths
372    void GetYearMonthDayForFiles(unsigned short& year, unsigned short& month, unsigned short& day);
373    ///Appends the relevant year month day to a given path
374    void AppendYearMonthDaytoPath(string& path);
375    ///Form the files path
376    string CompileFileName(const string &path, const string &service, const string & extension, const Time &time=Time());
377    ///Form the files path
378    string CompileFileName(const string &path, uint32_t run, const string &service, const string & extension, const Time &time=Time());
379    ///Check whether service is in black and/or white list
380    bool ShouldSubscribe(const string& server, const string& service);
381    ///Subscribe to a given server and service
382    DimStampedInfo* SubscribeToPlease(const string& server, const string& service);
383    ///Open a text file and checks for ofstream status
384    void OpenTextFilePlease(ofstream& stream, const string& name);
385    ///Check if a dir is . and returns the actual string corresponding to .
386//    string CheckIfDirIsDot(const string& dir);
387    ///Remembers the size of newly opened files. for statistic purposes
388    bool RememberFileOrigSizePlease(string& fileName, bool nightly);
389    ///Checks if the input osftream is in error state, and if so close it.
390    void CheckForOfstreamError(ofstream& out);
391    ///Checks if a given path exist
392    bool DoesPathExist(string path);
393public:
394    ///Create a given directory
395    bool CreateDirectory(string path);
396    /***************************************************
397    * INHERITED FROM DIMSERVICEINFOLIST
398    ***************************************************/
399    ///Add a new service subscription
400    void AddService(const string&, const string&, const string&, bool);
401    ///Remove a given service subscription
402    void RemoveService(const string&, const string&, bool);
403    ///Remove all the services associated with a given server
404    void RemoveAllServices(const string&);
405}; //DataLogger
406
407// --------------------------------------------------------------------------
408//
409//! Check if a given path exists
410//! @param path the path to be checked
411//! @return whether or not the creation has been successfull
412//
413bool DataLogger::CreateDirectory(string path)
414{
415    //remove last '/', if present
416    if (path[path.size()-1] == '/')
417        path = path.substr(0, path.size()-1);
418    //create boost path
419    boost::filesystem::path fullPath =  boost::filesystem::system_complete(boost::filesystem::path(path));
420
421    //if path does not exist, check if upper levels exist already
422    if (!boost::filesystem::exists(fullPath))
423    {
424        if (path.size() <= 1)
425        {//we're hitting "/", which SHOULD have existed...
426            Error("Something unexpected happened while creating a path");
427        }
428        CreateDirectory(path.substr(0, path.find_last_of('/')));
429    }
430    else
431    {//if path already exist, make sure it does not designate a file (filenames cannot be checked if they do not exist)
432        if (!boost::filesystem::is_directory(fullPath))
433        {
434            Error("Path to be created contains a file name: '" + path + "'");
435            return false;
436        }
437
438        return true;
439    }
440    //path does not exist, and upper level have been created already by recusrion.
441    mode_t rightsMask = S_IRWXU | S_IXGRP | S_IRGRP | S_IXOTH | S_IROTH; //everybody read, owner writes
442
443    int returnValue = mkdir(path.c_str(), rightsMask);
444
445    if (returnValue != 0)
446    {
447        ostringstream str;
448        str << "Could not create directory " << path << " mkdir error code: " << errno;
449        Error(str.str());
450        return false;
451    }
452
453    return true;
454}
455// --------------------------------------------------------------------------
456//
457//! Check if a given path exists
458//! @param path the path to be checked
459//! @return whether or not the given path exists
460//
461bool DataLogger::DoesPathExist(string path)
462{
463    boost::filesystem::path fullPath = boost::filesystem::system_complete(boost::filesystem::path(path));
464
465
466    if (!boost::filesystem::exists(fullPath))
467       return false;
468
469    if (!boost::filesystem::is_directory(fullPath))
470    {
471        Error("Path given for checking '" + path + "' designate a file name. Please provide a path name only");
472        return false;
473    }
474
475    if (access(path.c_str(), R_OK|W_OK|X_OK) != 0)
476    {
477        Error("Missing read, write or execute permissions on directory '" + path + "'");
478        return false;
479    }
480
481    return true;
482}
483// --------------------------------------------------------------------------
484//
485//! Add a new service subscription
486//! @param server the server for which the subscription should be created
487//! @param service the service for which the subscription should be created
488//! @param isCmd whether this is a Dim Command or not. Commands are not logged
489//
490void DataLogger::AddService(const string& server, const string& service, const string&, bool isCmd)
491{
492         //dataLogger does not subscribe to commands
493    if (isCmd)
494        return;
495
496    //check the given subscription against black and white lists
497    if (!ShouldSubscribe(server, service))
498        return;
499
500    map<string, SubscriptionType> &list = fServiceSubscriptions[server];
501
502    if (list.find(service) != list.end())
503    {
504        Error("Service " + server + "/" + service + " is already in the dataLogger's list. ignoring its update.");
505        return;
506    }
507
508    list[service].dimInfo = shared_ptr<DimStampedInfo>(SubscribeToPlease(server, service));
509    list[service].server  = server;
510    list[service].service = service;
511    fNumSubAndFitsData.numSubscriptions++;
512    if (fDebugIsOn)
513        Debug("Added subscription to " + server + "/" + service);
514}
515// --------------------------------------------------------------------------
516//
517//! Remove a given service subscription
518//! @param server the server for which the subscription should be removed
519//! @param service the service that should be removed
520//! @param isCmd whether or not this is a command
521//
522void DataLogger::RemoveService(const string& server, const string& service, bool isCmd)
523{
524    if (isCmd)
525        return;
526    if (fServiceSubscriptions[server].erase(service) != 1)
527    {
528        ostringstream str;
529        str << "Subscription " << server << "/" << service << "could not be removed as it is not present";
530        Error(str.str());
531        return;
532    }
533    fNumSubAndFitsData.numSubscriptions--;
534    if (fDebugIsOn)
535    {
536        Debug("Removed subscription to " + server + "/" + service);
537    }
538}
539// --------------------------------------------------------------------------
540//
541//! Remove all the services associated with a given server
542//! @param server the server for which all the services should be removed
543//
544void DataLogger::RemoveAllServices(const string& server)
545{
546    fNumSubAndFitsData.numSubscriptions -= fServiceSubscriptions[server].size();
547    fServiceSubscriptions[server].clear();
548    if (fDebugIsOn)
549    {
550        Debug("Removed all subscriptions to " + server + "/");
551    }
552}
553// --------------------------------------------------------------------------
554//
555//! Checks if the given ofstream is in error state and if so, close it
556//! @param out the ofstream that should be checked
557//
558void DataLogger::CheckForOfstreamError(ofstream& out)
559{
560    if (!out.good())
561    {
562        Error("An error occured while writing to a text file. Closing it");
563        if (out.is_open())
564            out.close();
565        SetCurrentState(kSM_WriteError);
566    }
567}
568// --------------------------------------------------------------------------
569//
570//! Checks the size on disk of a given size, and remembers it in the relevant member variable
571//! @param fileName the file for which the size on disk should be retrieved
572//! @param nightly whether this is a run or nightly file, so that its size is added to the correct member variable
573//
574bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly)
575{
576    //get the size of the file we're about to open
577    if (fFileSizesMap.find(fileName) != fFileSizesMap.end())
578        return false;
579
580    if (nightly)
581        fBaseSizeNightly += GetFileSize(fileName);
582    else
583        fBaseSizeRun += GetFileSize(fileName);
584    fFileSizesMap[fileName] = 0;
585    return true;
586}
587
588// --------------------------------------------------------------------------
589//
590//! Open a text file and checks for error code
591//! @param stream the ofstream for which the file should be opened
592//! @name the file name
593//
594void DataLogger::OpenTextFilePlease(ofstream& stream, const string& name)
595{
596    errno = 0;
597    stream.open(name.c_str(), ios_base::out | ios_base::app);
598    if (errno != 0)
599    {
600        ostringstream str;
601        str << "Unable to open file: " << name << ". Reason: " << strerror(errno) << " [" << errno << "]";
602        Error(str);
603    }
604}
605// --------------------------------------------------------------------------
606//
607//! Create a new dim subscription to a given server and service
608//! @param server the server name
609//! @param service the service name
610//
611DimStampedInfo* DataLogger::SubscribeToPlease(const string& server, const string& service)
612{
613    if (fDebugIsOn)
614    {
615        ostringstream str;
616        str << "Subscribing to service " << server << "/" << service;
617        Debug(str);
618    }
619    return new DimStampedInfo((server + "/" + service).c_str(), const_cast<char*>(""), this);
620}
621// --------------------------------------------------------------------------
622//
623//! Check whether a service should be subscribed to, based on the black/white list entries
624//! @param server the server name associated with the service being checked
625//! @param service the service name associated with the service being checked
626//
627bool DataLogger::ShouldSubscribe(const string& server, const string& service)
628{
629    if (fWhiteList.size()>0 &&
630        (fWhiteList.find(server + "/") == fWhiteList.end()) &&
631        (fWhiteList.find(server + "/" + service) == fWhiteList.end()) &&
632        (fWhiteList.find("/" + service) == fWhiteList.end()))
633        return false;
634
635    if ((fBlackList.find(server + "/") != fBlackList.end()) ||
636         (fBlackList.find(server + "/" + service) != fBlackList.end()) ||
637         (fBlackList.find("/" + service) != fBlackList.end()))
638        return false;
639
640    return true;
641}
642// --------------------------------------------------------------------------
643//
644//! Compiles a file name
645//! @param path the base path where to put the file
646//! @param time the time at which the file is created
647//! @param service the service name, if any
648//! @param extension the extension to add, if any
649//
650string DataLogger::CompileFileName(const string &path, const string &service, const string & extension, const Time &time)
651{
652       ostringstream str;
653       //calculate time suitable for naming files.
654       const Time ftime(time-boost::posix_time::time_duration(12,0,0));
655
656       //output it
657       str << path << Time::fmt("/%Y/%m/%d") << ftime;
658
659       //check if target directory exist
660       if (!DoesPathExist(str.str()))
661           CreateDirectory(str.str());
662
663       //output base of file name
664       str << Time::fmt("/%Y_%m_%d") << ftime;
665
666       //output service name
667       if (!service.empty())
668           str  << "_" << service;
669
670       //output appropriate extension
671       if (!extension.empty())
672           str << "." << extension;
673
674       return str.str();
675}
676// --------------------------------------------------------------------------
677//
678//! Compiles a file name
679//! @param path the base path where to put the file
680//! @param time the time at which the file is created
681//! @param run the run number
682//! @param service the service name, if any
683//! @param extension the extension to add, if any
684//
685string DataLogger::CompileFileName(const string &path, uint32_t run, const string &service, const string & extension, const Time &time)
686{
687       ostringstream str;
688       //calculate suitable time for naming files and output it
689       str << path << Time::fmt("/%Y/%m/%d") << (time-boost::posix_time::time_duration(12,0,0));
690
691       //check if target directory exist
692       if (!DoesPathExist(str.str()))
693           CreateDirectory(str.str());
694
695       //output base of file name
696       str << '/' << setfill('0') << setw(8) << run;
697
698       //output service name
699       if (!service.empty())
700           str  << "_" << service;
701
702       //output appropriate extension
703       if (!extension.empty())
704           str << "." << extension;
705       return str.str();
706}
707
708// --------------------------------------------------------------------------
709//
710//!retrieves the size on disk of a file
711//! @param fileName the full file name for which the size on disk should be retrieved
712//! @return the size of the file on disk, in bytes. 0 if the file does not exist or if an error occured
713//
714off_t DataLogger::GetFileSize(const string& fileName)
715{
716    errno = 0;
717    struct stat st;
718    if (!stat(fileName.c_str(), &st))
719        return st.st_size;
720
721    if (errno != 0 && errno != 2)//ignoring error #2: no such file or directory is not an error for new files
722    {
723        ostringstream str;
724        str << "Unable to stat " << fileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
725        Error(str);
726    }
727
728    return 0;
729}
730// --------------------------------------------------------------------------
731//
732//! Removes the oldest run number and closes the fits files that should be closed
733//! Also creates the fits grouping file
734//
735void DataLogger::RemoveOldestRunNumber()
736{
737    if (fDebugIsOn)
738    {
739        ostringstream str;
740        str << "Removing run number " << fRunNumber.front().runNumber;
741        Debug(str);
742    }
743    CreateFitsGrouping(fRunNumber.front().openedFits, fRunNumber.front().runNumber);
744
745    //crawl through the subscriptions to see if there are still corresponding fits files opened.
746    for (SubscriptionsListType::iterator x=fServiceSubscriptions.begin();
747         x!=fServiceSubscriptions.end(); x++)
748        for (map<string, SubscriptionType>::iterator y=x->second.begin();
749             y!=x->second.end(); y++)
750            if (y->second.runFile.fRunNumber == fRunNumber.front().runNumber && y->second.runFile.IsOpen())
751            {
752                if (fDebugIsOn)
753                {
754                    ostringstream str;
755                    str << "Closing Fits run file " << y->second.runFile.fFileName;
756                    Debug(str);
757                }
758                y->second.runFile.Close();
759            }
760    //if a grouping file is on, decrease the number of opened fits manually
761    if (fRunNumber.front().runFitsFile)
762        (fNumSubAndFitsData.numOpenFits)--;
763    //remove the entry
764    fRunNumber.pop_front();
765}
766
767// --------------------------------------------------------------------------
768//
769//! Calculate the total number of written bytes since the logger was started
770//! @param statVar the data structure that should be updated
771//! @param shouldWarn whether or not error messages should be outputted
772//! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information
773//
774bool DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting)
775{
776    if (!isPrinting)
777        fStatsMutex.lock();
778#ifdef HAVE_FITS
779    if (isPrinting)
780    {
781        ostringstream str;
782        str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:";
783        Message(str);
784    }
785
786    ///TODO the grouping file is dealt with several times. This should not be a problem but well, better to fix it I guess.
787    for (SubscriptionsListType::const_iterator x=fServiceSubscriptions.begin();
788         x!=fServiceSubscriptions.end(); x++)
789    {
790        for (map<string, SubscriptionType>::const_iterator y=x->second.begin();
791             y!=x->second.end(); y++)
792        {
793            if (y->second.runFile.IsOpen())
794            {
795                    fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();
796                    if (isPrinting)
797                        Message("-> "+y->second.runFile.fFileName);
798            }
799            if (y->second.nightlyFile.IsOpen())
800            {
801                fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();
802                if (isPrinting)
803                    Message("-> "+y->second.nightlyFile.fFileName);
804            }
805        }
806    }
807#else
808    if (isPrinting)
809        Message("FITS output disabled at compilation");
810#endif
811    //gather log and report files sizes on disk
812    if (fNightlyLogFile.is_open())
813        fFileSizesMap[fFullNightlyLogFileName] = GetFileSize(fFullNightlyLogFileName);
814    if (fNightlyReportFile.is_open())
815        fFileSizesMap[fFullNightlyReportFileName] = GetFileSize(fFullNightlyReportFileName);
816    for (list<RunNumberType>::iterator it = fRunNumber.begin(); it != fRunNumber.end(); it++)
817    {
818        if (it->reportFile->is_open())
819            fFileSizesMap[it->reportName] = GetFileSize(it->reportName);
820#ifdef RUN_LOGS
821        if (it->logFile->is_open())
822            fFileSizesMap[it->logName] = GetFileSize(it->logName);
823#endif
824    }
825
826    bool shouldWarn = false;
827    struct statvfs vfs;
828    if (!statvfs(fNightlyFilePath.c_str(), &vfs))
829        statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
830    else
831    {
832        ostringstream str;
833        str << "Unable to retrieve stats for " << fNightlyFilePath << ". Reason: " << strerror(errno) << " [" << errno << "]";
834        if (!shouldWarn)
835            Error(str);
836        statVar.freeSpace = -1;
837    }
838    //sum up all the file sizes. past and present
839    statVar.sizeWritten = 0;
840    for (map<string, long>::const_iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end();  it++)
841        statVar.sizeWritten += it->second;
842    statVar.sizeWritten -= fBaseSizeNightly;
843    statVar.sizeWritten -= fBaseSizeRun;
844
845    if (!isPrinting)
846        fStatsMutex.unlock();
847
848    return shouldWarn;
849}
850//static members initialization
851//since I do not check the transition/config names any longer, indeed maybe these could be hard-coded... but who knows what will happen in the future ?
852const char* DataLogger::fConfigDay = "CONFIG_DAY";
853const char* DataLogger::fConfigRun = "CONFIG_RUN";
854const char* DataLogger::fConfigRunNumber = "CONFIG_RUN_NUMBER";
855const char* DataLogger::fConfigLog = "LOG";
856const char* DataLogger::fTransStart = "START";
857const char* DataLogger::fTransStop = "STOP";
858const char* DataLogger::fTransStartRun = "START_RUN";
859const char* DataLogger::fTransStopRun = "STOP_RUN";
860const char* DataLogger::fTransReset = "RESET";
861const char* DataLogger::fTransWait = "WAIT_RUN_NUMBER";
862const char* DataLogger::fRunNumberInfo = "RUN_NUMBER";
863const char* DataLogger::fPrintCommand = "PRINT";
864const char* DataLogger::fDebugOnOff = "DEBUG";
865const char* DataLogger::fStatsPeriod = "STATS_PERIOD";
866const char* DataLogger::fStartStopOpenedFiles = "OPENED_FILES_SRVC";
867const char* DataLogger::fStartStopNumSubsAndFits = "NUM_SUBS_SRVC";
868
869// --------------------------------------------------------------------------
870//
871//! Monitor the number of opened files and total size written, and distributes this data through a Dim service
872//
873//
874void DataLogger::ServicesMonitoring()
875{
876       struct statvfs vfs;
877        if (!statvfs(fNightlyFilePath.c_str(), &vfs))
878            fStatVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
879        else
880            fStatVar.freeSpace = -1;
881
882        DimDescribedService srvc ("DATA_LOGGER/STATS", "X:3", fStatVar, "Add description here");
883        fPreviousSize = 0;
884        //loop-wait for broadcast
885        while (fContinueMonitoring)
886        {
887            if (fStatsPeriodDuration == 0.0f)
888            {
889                sleep(0.1f);
890                continue;
891            }
892
893            sleep(fStatsPeriodDuration);
894
895
896            fStatsMutex.lock();
897
898            if (fStatVar.writingRate != 0) //if data has been written
899            {
900                srvc.updateService();
901
902                if(fDebugIsOn)
903                {
904                    ostringstream str;
905                    str << "Size written: " << fStatVar.sizeWritten/1000 << " kB; writing rate: ";
906                    str << fStatVar.writingRate/1000 << " kB/s; free space: ";
907                    str << fStatVar.freeSpace/(1000*1000) << " MB";
908                    Debug(str);
909                }
910            }
911            fStatsMutex.unlock();
912        }
913}
914
915// --------------------------------------------------------------------------
916//
917//! Default constructor. The name of the machine is given DATA_LOGGER
918//! and the state is set to kSM_Ready at the end of the function.
919//
920//!Setup the allows states, configs and transitions for the data logger
921//
922DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER")
923{
924    //initialize member data
925    fNightlyFilePath = ".";
926    fRunFilePath = ".";
927
928    //Give a name to this machine's specific states
929    AddStateName(kSM_NightlyOpen,      "NightlyFileOpen",  "The summary files for the night are open.");
930    AddStateName(kSM_WaitingRun,       "WaitForRun",       "The summary files for the night are open and we wait for a run to be started.");
931    AddStateName(kSM_Logging,          "Logging",          "The summary files for the night and the files for a single run are open.");
932    AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "The folder for the nighly summary files is invalid.");
933    AddStateName(kSM_BadRunConfig,     "ErrRunFolder",     "The folder for the run files is invalid.");
934    AddStateName(kSM_WriteError, "ErrWrite", "An error occured while writing to a file.");
935
936    /*Add the possible transitions for this machine*/
937    AddEvent(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig)
938            (boost::bind(&DataLogger::StartPlease, this))
939            ("Start the nightly logging. Nightly file location must be specified already");
940
941    AddEvent(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging, kSM_WriteError)
942            (boost::bind(&DataLogger::GoToReadyPlease, this))
943            ("Stop all data logging, close all files.");
944
945    AddEvent(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig)
946            (boost::bind(&DataLogger::StartRunPlease, this))
947            ("Start the run logging. Run file location must be specified already.");
948
949    AddEvent(kSM_WaitingRun, fTransStopRun, kSM_Logging)
950            (boost::bind(&DataLogger::StopRunPlease, this))
951            ("Wait for a run to be started, open run-files as soon as a run number arrives.");
952
953    AddEvent(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_WriteError)
954            (boost::bind(&DataLogger::GoToReadyPlease, this))
955            ("Transition to exit error states. Closes the nightly file if already opened.");
956
957    AddEvent(kSM_WaitingRun, fTransWait, kSM_NightlyOpen)
958            (boost::bind(&DataLogger::NightlyToWaitRunPlease, this));
959
960    /*Add the possible configurations for this machine*/
961    AddEvent(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig)
962            (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1))
963            ("Configure the folder for the nightly files."
964             "|Path[string]:Absolute or relative path name where the nightly files should be stored.");
965
966    AddEvent(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
967            (boost::bind(&DataLogger::ConfigureRunFileName, this, _1))
968            ("Configure the folder for the run files."
969             "|Path[string]:Absolute or relative path name where the run files should be stored.");
970
971    AddEvent(fConfigRunNumber, "X", kSM_Ready, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig, kSM_Logging)
972            (boost::bind(&DataLogger::ConfigureRunNumber, this, _1))
973            ("Configure the run number. Cannot be done in logging state");
974
975     //Provide a print command
976     ostringstream str;
977     str <<     kSM_Ready << " " << kSM_NightlyOpen << " " << kSM_WaitingRun << " " << kSM_Logging << " " << kSM_BadNightlyConfig << " " << kSM_WriteError;
978     str << " " << kSM_BadRunConfig;
979     AddEvent(fPrintCommand, str.str().c_str(), "")
980             (boost::bind(&DataLogger::PrintStatePlease, this, _1))
981             ("Print information about the internal status of the data logger.");
982
983     OpenFileToDim fToDim;
984     fToDim.code = 0;
985     fToDim.fileName[0] = '\0';
986
987     fOpenedNightlyFiles = new DimDescribedService(GetName() + "/FILENAME_NIGHTLY", "I:1;C", fToDim,
988                               "Path and base name which is used to compile the filenames for the nightly files."
989                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
990                               "|Name[string]:path and base file name");
991
992     fOpenedRunFiles = new DimDescribedService(GetName() + "/FILENAME_RUN", "I:1;C", fToDim,
993                               "Path and base name which is used to compile the filenames for the run files."
994                               "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
995                               "|Name[string]:path and base file name");
996
997     fNumSubAndFitsData.numSubscriptions = 0;
998     fNumSubAndFitsData.numOpenFits = 0;
999     fNumSubAndFits = new DimDescribedService(GetName() + "/NUM_SUBS", "I:2", fNumSubAndFitsData,
1000                               "Shows number of services to which the data logger is currently subscribed and the total number of open files."
1001                               "|Subscriptions[int]:number of dim services to which the data logger is currently subscribed."
1002                               "|NumOpenFiles[int]:number of files currently open by the data logger");
1003
1004     //services parameters
1005     fDebugIsOn = false;
1006     fStatsPeriodDuration = 1.0f;
1007     fOpenedFilesIsOn = true;
1008     fNumSubAndFitsIsOn = true;
1009
1010     //provide services control commands
1011     AddEvent(fDebugOnOff, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1012             (boost::bind(&DataLogger::SetDebugOnOff, this, _1))
1013             ("Switch debug mode on off. Debug mode prints ifnormation about every service written to a file."
1014              "|Enable[bool]:Enable of disable debug mode (yes/no).");
1015
1016     AddEvent(fStatsPeriod, "F", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1017             (boost::bind(&DataLogger::SetStatsPeriod, this, _1))
1018             ("Interval in which the data-logger statitistics service (STATS) is updated."
1019              "Interval[s]:Floating point value in seconds.");
1020
1021     AddEvent(fStartStopOpenedFiles, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1022              (boost::bind(&DataLogger::SetOpenedFilesOnOff ,this, _1))
1023              ("Switch off the service which distributes information about the open files.");
1024
1025     AddEvent(fStartStopNumSubsAndFits, "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
1026             (boost::bind(&DataLogger::SetNumSubsAndFitsOnOff, this, _1))
1027             ("Switch off the service which distributes information about the number of subscriptions and open files.");
1028
1029     fDestructing = false;
1030
1031     //start the monitoring service
1032     fStatVar.sizeWritten = 0;
1033     fStatVar.freeSpace = 0;
1034     fStatVar.writingRate = 0;
1035     fPreviousStatsUpdateTime = Time().Mjd();
1036     fPreviousOldRunNumberCheck = Time().Mjd();
1037     fContinueMonitoring = true;
1038     fMonitoringThread = boost::thread(boost::bind(&DataLogger::ServicesMonitoring, this));
1039     fBaseSizeNightly = 0;
1040     fBaseSizeRun = 0;
1041     fDailyFileDayChangedAlready = true;
1042     if(fDebugIsOn)
1043     {
1044         Debug("DataLogger Init Done.");
1045     }
1046}
1047
1048// --------------------------------------------------------------------------
1049//
1050//! Destructor
1051//
1052DataLogger::~DataLogger()
1053{
1054    if (fDebugIsOn)
1055    {
1056        Debug("DataLogger destruction starts");   
1057    }
1058    fDestructing = true;
1059    //first let's go to the ready state
1060    GoToReadyPlease(); 
1061    //release the services subscriptions
1062    fServiceSubscriptions.clear();
1063    //exit the monitoring loop
1064    fContinueMonitoring = false;
1065
1066    fMonitoringThread.join();
1067    //clear any remaining run number (should remain only one)
1068     while (fRunNumber.size() > 0)
1069     {
1070         RemoveOldestRunNumber();
1071     }
1072
1073    delete fOpenedNightlyFiles;
1074    delete fOpenedRunFiles;
1075    delete fNumSubAndFits;
1076
1077    if (fDebugIsOn)
1078    {
1079        Debug("DataLogger desctruction ends");   
1080    }
1081}
1082
1083// --------------------------------------------------------------------------
1084//
1085//! Inherited from DimInfo. Handles all the Infos to which we subscribed, and log them
1086//
1087void DataLogger::infoHandler()
1088{
1089    // Make sure getTimestamp is called _before_ getTimestampMillisecs
1090    if (fDestructing)
1091        return;
1092
1093    DimInfo* I = getInfo();
1094
1095    if (I==NULL)
1096        return;
1097    //check if the service pointer corresponds to something that we subscribed to
1098    //this is a fix for a bug that provides bad Infos when a server starts
1099    bool found = false;
1100    SubscriptionsListType::iterator x;
1101    map<string, SubscriptionType>::iterator y;
1102     for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
1103    {//find current service is subscriptions
1104        for (y=x->second.begin(); y!=x->second.end();y++)
1105            if ((y->second.dimInfo).get() == I)
1106            {
1107                found = true;   
1108                break;
1109            }
1110        if (found)
1111            break;
1112    }
1113    if (!found)
1114    {
1115        DimServiceInfoList::infoHandler();
1116        return;
1117    }
1118    if (I->getSize() <= 0)
1119        return;
1120
1121        // Make sure that getTimestampMillisecs is NEVER called before
1122        // getTimestamp is properly called
1123        // check that the message has been updated by something, i.e. must be different from its initial value
1124    if (I->getTimestamp() == 0)
1125        return;
1126
1127    CheckForRunNumber(I);
1128
1129    ReportPlease(I, y->second);
1130
1131    //update the fits files sizes
1132    Time cTime = Time();
1133    if ((fStatsPeriodDuration != 0) && ((cTime.Mjd() - fPreviousStatsUpdateTime)*24*60*60 > fStatsPeriodDuration))
1134    {
1135        calculateTotalSizeWritten(fStatVar, false);
1136        fStatVar.writingRate = (fStatVar.sizeWritten - fPreviousSize)/((cTime.Mjd() - fPreviousStatsUpdateTime)*24*60*60);
1137        fPreviousSize = fStatVar.sizeWritten;
1138        fPreviousStatsUpdateTime = cTime.Mjd();
1139    }
1140    if ((cTime.Mjd() - fPreviousOldRunNumberCheck)*24*60*60 > 10.0)
1141    {
1142        while (fRunNumber.size() > 1 && (cTime - fRunNumber.back().time) > boost::posix_time::time_duration(0,0,10,0))
1143        {
1144             RemoveOldestRunNumber();
1145        }
1146        fPreviousOldRunNumberCheck = cTime.Mjd();
1147    }
1148}
1149// --------------------------------------------------------------------------
1150//
1151//! Open the text files associated with the given run number
1152//! @param run the run number to be dealt with
1153//
1154int DataLogger::OpenRunFile(RunNumberType& run)
1155{
1156#ifdef RUN_LOGS
1157    if (run.logFile->is_open())
1158    {
1159        ostringstream str;
1160        str << "Log file " << run.logName << " was already open when trying to open it in OpenRunFile";
1161        Error(str);
1162        return -1;
1163    }
1164    run.logName = CompileFileName(fRunFilePath, run.runNumber, "", "log");
1165    errno = 0;
1166    run.logFile->open(run.logName.c_str(), ios_base::out | ios_base::app);
1167    if (errno != 0)
1168    {
1169        ostringstream str;
1170        str << "Unable to open run Log " << run.logName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1171        Error(str);
1172    }
1173#endif
1174    //open report file
1175    run.reportName = CompileFileName(fRunFilePath, run.runNumber, "", "rep");
1176    if (run.reportFile->is_open())
1177    {
1178        ostringstream str;
1179        str << "Report file " << run.reportName << " was already open when trying to open it in OpenRunFile";
1180        Error(str);
1181        return -1;
1182    }
1183    errno = 0;
1184    run.reportFile->open(run.reportName.c_str(), ios_base::out | ios_base::app);
1185    if (errno != 0)
1186    {
1187        ostringstream str;
1188        str << "Unable to open run report " << run.reportName << ". Reason: " << strerror(errno) << " [" << errno << "]";
1189        Error(str);
1190    }
1191
1192#ifdef RUN_LOGS
1193    if (!run.logFile->is_open() || !run.reportFile->is_open())
1194#else
1195    if (!run.reportFile->is_open())
1196#endif
1197    {
1198        ostringstream str;
1199        str << "Something went wrong while openning files ";
1200#ifdef RUN_LOGS
1201        str << run.logName << " and ";
1202#endif
1203        str << run.reportName;
1204        Error(str);
1205        return -1;
1206    }
1207    //get the size of the newly opened file.
1208#ifdef RUN_LOGS
1209    RememberFileOrigSizePlease(run.logName, false);
1210#endif
1211    RememberFileOrigSizePlease(run.reportName, false);
1212
1213    //TODO this notification scheme might be messed up now.... fix it !
1214    string baseFileName = CompileFileName(fRunFilePath, run.runNumber, "", "");
1215    NotifyOpenedFile(baseFileName, 3, fOpenedRunFiles);
1216    run.openedFits.clear();
1217    return 0;
1218}
1219// --------------------------------------------------------------------------
1220//
1221//! Add a new active run number
1222//! @param newRun the new run number
1223//! @param time the time at which the new run number was issued
1224//
1225void DataLogger::AddNewRunNumber(int64_t newRun, Time time)
1226{
1227
1228    if (newRun > 0xffffffff)
1229    {
1230        Error("New run number too large, out of range. Ignoring.");
1231        return;
1232    }
1233    if (fDebugIsOn)
1234    {
1235        ostringstream str;
1236        str << "Adding new run number " << newRun << " that was issued on " << time;
1237        Debug(str);
1238    }
1239    //Add new run number to run number list
1240    fRunNumber.push_back(RunNumberType());
1241    fRunNumber.back().runNumber = uint32_t(newRun);
1242    fRunNumber.back().time = time;
1243
1244    ostringstream str;
1245    str << "The new run number is: " << fRunNumber.back().runNumber;
1246    Message(str);
1247
1248    if (GetCurrentState() != kSM_Logging)
1249        return;
1250    //open the log and report files
1251    OpenRunFile(fRunNumber.back());
1252}
1253// --------------------------------------------------------------------------
1254//
1255//! Checks whether or not the current info is a run number.
1256//! If so, then remember it. A run number is required to open the run-log file
1257//! @param I
1258//!        the current DimInfo
1259//
1260void DataLogger::CheckForRunNumber(DimInfo* I)
1261{
1262    if (strstr(I->getName(), fRunNumberInfo) != NULL)
1263    {//assumes that the run number is an integer
1264        //check if some run number entries can be deleted leave one so that two remain after adding the new one
1265        AddNewRunNumber(I->getLonglong(), Time(I->getTimestamp(), I->getTimestampMillisecs()*1000));
1266    }
1267}
1268
1269// --------------------------------------------------------------------------
1270//
1271//! write infos to log files.
1272//! @param I
1273//!     The current DimInfo
1274//! @param sub
1275//!        The dataLogger's subscription corresponding to this DimInfo
1276//
1277void DataLogger::ReportPlease(DimInfo* I, SubscriptionType& sub)
1278{
1279
1280
1281    //should we log or report this info ? (i.e. is it a message ?)
1282    bool isItaReport = ((strstr(I->getName(), "Message") == NULL) && (strstr(I->getName(), "MESSAGE") == NULL));
1283    if (I->getFormat()[0] == 'C')
1284        isItaReport = false;
1285
1286    if (!fNightlyReportFile.is_open())
1287        return;
1288
1289    //Check whether we should close and reopen daily text files or not
1290    //This should work in any case base of the following:
1291    // - fDailyFileDayChangedAlready is initialized to true. So if the dataLogger is started around noon, no file will be closed
1292    // - fDailyFileDayChangedAlready is set to false if (time != 12), so the file will be closed and reopened only if the logger runs since before noon (which is the required behavior)
1293    //This only applies to text files. Fits are closed and reopened based on the last and current service received time.
1294    //this was not applicable to text files, because as they gather several services, we have no guarantee that the received time will be greater than the previous one,
1295    //which could lead to several close/reopen instead of only one.
1296    if (Time().h() == 12 && !fDailyFileDayChangedAlready)
1297    {
1298        if (fDebugIsOn)
1299            Debug("Its Noon! Closing and reopening daily text files");
1300
1301        fNightlyLogFile.close();
1302        fNightlyReportFile.close();
1303
1304        fFullNightlyLogFileName = CompileFileName(fNightlyFilePath, "", "log");
1305        OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName);
1306
1307        fFullNightlyReportFileName = CompileFileName(fNightlyFilePath, "", "rep");
1308        OpenTextFilePlease(fNightlyReportFile, fFullNightlyReportFileName);
1309
1310        fDailyFileDayChangedAlready = true;
1311    }
1312    if (Time().h() != 12 && fDailyFileDayChangedAlready)
1313        fDailyFileDayChangedAlready = false;
1314
1315    //create the converter for that service
1316    if ((!sub.fConv.get()) && isItaReport)
1317    {
1318        //trick the converter in case of 'C'. why do I do this ? well simple: the converter checks that the right number
1319        //of bytes was written. because I skip 'C' with fits, the bytes will not be allocated, hence the "size copied ckeck"
1320        //of the converter will fail, hence throwing an exception.
1321        string fakeFormat(I->getFormat());
1322        if (fakeFormat[fakeFormat.size()-1] == 'C')
1323            fakeFormat = fakeFormat.substr(0, fakeFormat.size()-1);
1324        sub.fConv = shared_ptr<Converter>(new Converter(Out(), I->getFormat()));
1325        if (!sub.fConv)
1326        {
1327            ostringstream str;
1328            str << "Couldn't properly parse the format... service " << sub.dimInfo->getName() << " ignored.";
1329            Error(str);
1330            return;   
1331        }
1332    }
1333    //construct the header
1334    ostringstream header;
1335    const Time cTime(I->getTimestamp(), I->getTimestampMillisecs()*1000);
1336    fQuality = I->getQuality();
1337    fMjD = cTime.Mjd();
1338
1339    //figure out which run file should be used
1340    ofstream* targetRunFile = NULL;
1341    RunNumberType* cRunNumber = NULL;
1342    if (GetCurrentState() == kSM_Logging)
1343    {
1344        list<RunNumberType>::reverse_iterator rit;
1345        for (rit=fRunNumber.rbegin(); rit!=fRunNumber.rend(); rit++)
1346        {
1347            if (rit->time < cTime) //this is the run number that we want to use
1348            {
1349                //Find something better to convert iterator to pointer than the ugly line below....
1350                cRunNumber = &(*rit);
1351                sub.runNumber = rit->runNumber;
1352#ifdef RUN_LOGS
1353                targetRunFile = isItaReport ? (rit->reportFile).get() : (rit->logFile).get();
1354#else
1355                targetRunFile = isItaReport ? (rit->reportFile).get() : NULL;
1356#endif
1357                break;
1358            }
1359        }
1360        if (rit == fRunNumber.rend() && fRunNumber.size() != 0)
1361        {
1362            ostringstream str;
1363            str << "Could not find an appropriate run number for info coming at time: " << cTime;
1364            Error(str);
1365            Error("Active run numbers: ");
1366            for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
1367            {
1368                str.str("");
1369                str << rit->runNumber;
1370                Error(str);
1371            }
1372
1373        }
1374    }
1375
1376    if (isItaReport)
1377    {
1378        //write text header
1379        header << I->getName() << " " << fQuality << " ";
1380        header << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
1381        header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
1382        header << cTime.ms() << " " << I->getTimestamp() << " ";
1383
1384        string text;
1385        try
1386        {
1387            text = sub.fConv->GetString(I->getData(), I->getSize());
1388        }
1389        catch (const runtime_error &e)
1390        {
1391            Out() << kRed << e.what() << endl;
1392            ostringstream str;
1393            str << "Could not properly parse the data for service " << sub.dimInfo->getName();
1394            str << " reason: " << e.what() << ". Entry ignored";
1395            Error(str);
1396            return;
1397        }
1398
1399        if (text.empty())
1400        {
1401            ostringstream str;
1402            str << "Service " << sub.dimInfo->getName() << " sent an empty string";
1403            Info(str);
1404            return;
1405        }
1406        //replace bizarre characters by white space
1407        replace(text.begin(), text.end(), '\n', '\\');
1408        replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
1409       
1410        //write entry to Nightly report
1411        if (fNightlyReportFile.is_open())
1412        {
1413            fNightlyReportFile << header.str() << text << endl;
1414            CheckForOfstreamError(fNightlyReportFile);
1415        }
1416        //write entry to run-report
1417        if (targetRunFile && targetRunFile->is_open())
1418        {
1419            *targetRunFile << header.str() << text << endl;
1420            CheckForOfstreamError(*targetRunFile);
1421        }
1422    }
1423    else
1424    {//write entry to both Nightly and run logs
1425        string n = I->getName();
1426        ostringstream msg;
1427        msg << n << ": " << I->getString();
1428
1429        if (fNightlyLogFile.is_open())
1430        {
1431            MessageImp nightlyMess(fNightlyLogFile);
1432            nightlyMess.Write(cTime, msg.str().c_str(), fQuality);
1433            CheckForOfstreamError(fNightlyLogFile);
1434        }
1435        if (targetRunFile && targetRunFile->is_open())
1436        {
1437            MessageImp runMess(*targetRunFile);
1438            runMess.Write(cTime, msg.str().c_str(), fQuality);
1439            CheckForOfstreamError(*targetRunFile);
1440        }
1441    }
1442
1443#ifdef HAVE_FITS
1444    if (isItaReport)
1445    {
1446        //check if the last received event was before noon and if current one is after noon.
1447        //if so, close the file so that it gets reopened.
1448        if (sub.nightlyFile.IsOpen())
1449            if ((sub.lastReceivedEvent != Time::None) && (sub.lastReceivedEvent.h() < 12) && (cTime.h() >= 12))
1450            {
1451                sub.nightlyFile.Close();
1452            }
1453        sub.lastReceivedEvent = cTime;
1454        if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen() || sub.runNumber != sub.runFile.fRunNumber)
1455            OpenFITSFilesPlease(sub, cRunNumber);
1456        WriteToFITS(sub);
1457    }   
1458#endif
1459
1460}
1461
1462// --------------------------------------------------------------------------
1463//
1464//! print the dataLogger's current state. invoked by the PRINT command
1465//! @param evt
1466//!        the current event. Not used by the method
1467//! @returns
1468//!        the new state. Which, in that case, is the current state
1469//!
1470int DataLogger::PrintStatePlease(const Event& )
1471{
1472    Message("------------------------------------------");
1473    Message("------- DATA LOGGER CURRENT STATE --------");
1474    Message("------------------------------------------");
1475
1476    //print the path configuration
1477    Message("Nightly Path: " + boost::filesystem::system_complete(boost::filesystem::path(fNightlyFilePath)).directory_string());
1478    Message("Run Path: " + boost::filesystem::system_complete(boost::filesystem::path(fRunFilePath)).directory_string());
1479
1480    //print active run numbers
1481    ostringstream str;
1482    str << "Active Run Numbers:";
1483    for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
1484        str << " " << it->runNumber;
1485    if (fRunNumber.size()==0)
1486        str << " <none>";
1487    Message(str);
1488
1489    //print all the open files.
1490    Message("------------ OPENED FILES ----------------");
1491    if (fNightlyLogFile.is_open())
1492        Message("Nightly log-file: OPEN");
1493
1494    if (fNightlyReportFile.is_open())
1495        Message("Nightly report-file: OPEN");
1496
1497    for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
1498    {
1499#ifdef RUN_LOGS
1500        if (it->logFile->is_open())
1501            Message("Run log-file: " + it->logName + " (OPEN)");
1502#endif
1503        if (it->reportFile->is_open())
1504            Message("Run report-file: " + it->reportName + " (OPEN)");
1505    }
1506
1507    DataLoggerStats statVar;
1508    /*const bool statWarning =*/ calculateTotalSizeWritten(statVar, true);
1509
1510    Message("----------------- STATS ------------------");
1511    str.str("");
1512    str << "Total Size written: " << statVar.sizeWritten << " bytes.";
1513        Message(str);
1514    str.str("");
1515    str << "Disk free space:    " << statVar.freeSpace   << " bytes.";
1516        Message(str);
1517    str.str("");
1518    str << "Statistics are updated every " << fStatsPeriodDuration << " seconds";
1519    if (fStatsPeriodDuration != 0)
1520        Message(str);
1521    else
1522        Message("Statistics updates are currently disabled");
1523
1524    Message("------------ DIM SUBSCRIPTIONS -----------");
1525    str.str("");
1526    str << "There are " << fNumSubAndFitsData.numSubscriptions << " active DIM subscriptions.";
1527    Message(str);
1528    for (map<const string, map<string, SubscriptionType> >::const_iterator it=fServiceSubscriptions.begin(); it!= fServiceSubscriptions.end();it++)
1529    {
1530        Message("Server "+it->first);
1531        for (map<string, SubscriptionType>::const_iterator it2=it->second.begin(); it2!=it->second.end(); it2++)
1532            Message(" -> "+it2->first);
1533    }
1534    Message("--------------- BLOCK LIST ---------------");
1535    for (set<string>::const_iterator it=fBlackList.begin(); it != fBlackList.end(); it++)
1536        Message(" -> "+*it);
1537    if (fBlackList.size()==0)
1538        Message(" <empty>");
1539
1540    Message("--------------- ALLOW LIST ---------------");
1541    for (set<string>::const_iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++)
1542        Message(" -> "+*it);
1543    if (fWhiteList.size()==0)
1544        Message(" <empty>");
1545
1546    Message("-------------- GROUPING LIST -------------");
1547    Message("The following servers and/or services will");
1548    Message("be grouped into a single fits file:");
1549    for (set<string>::const_iterator it=fGrouping.begin(); it != fGrouping.end(); it++)
1550        Message(" -> "+*it);
1551    if (fGrouping.size()==0)
1552        Message(" <no grouping>");
1553
1554    Message("------------------------------------------");
1555    Message("-------- END OF DATA LOGGER STATE --------");
1556    Message("------------------------------------------");
1557
1558    return GetCurrentState();
1559}
1560
1561// --------------------------------------------------------------------------
1562//
1563//! turn debug mode on and off
1564//! @param evt
1565//!        the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1
1566//! @returns
1567//!        the new state. Which, in that case, is the current state
1568//!
1569int DataLogger::SetDebugOnOff(const Event& evt)
1570{
1571    bool backupDebug = fDebugIsOn;
1572    fDebugIsOn = evt.GetBool();
1573    if (fDebugIsOn == backupDebug)
1574        Warn("Warning: debug mode was already in the requested state");
1575    else
1576    {
1577        ostringstream str;
1578        str << "Debug mode is now " << fDebugIsOn;
1579        Message(str);
1580    }
1581    return GetCurrentState();
1582}
1583// --------------------------------------------------------------------------
1584//
1585//! set the statistics update period duration. 0 disables the statistics
1586//! @param evt
1587//!        the current event. contains the new duration.
1588//! @returns
1589//!        the new state. Which, in that case, is the current state
1590//!
1591int DataLogger::SetStatsPeriod(const Event& evt)
1592{
1593    float backupDuration = fStatsPeriodDuration;
1594    fStatsPeriodDuration = evt.GetFloat();
1595    if (fStatsPeriodDuration < 0)
1596    {
1597        Error("Statistics period duration should be greater than zero. Discarding provided value.");
1598        fStatsPeriodDuration = backupDuration;
1599        return GetCurrentState();   
1600    }
1601    if (!finite(fStatsPeriodDuration))// != fStatsPeriodDuration)
1602    {
1603        Error("Provided duration does not appear to be a valid float. discarding it.");
1604        fStatsPeriodDuration = backupDuration;
1605        return GetCurrentState();   
1606    }
1607    if (backupDuration == fStatsPeriodDuration)
1608        Warn("Warning: statistics period was not modified: supplied value already in use");
1609    else
1610    {
1611        if (fStatsPeriodDuration == 0.0f)
1612            Message("Statistics are now OFF");
1613        else
1614        {
1615            ostringstream str;
1616            str << "Statistics period is now " << fStatsPeriodDuration << " seconds";
1617            Message(str);
1618        }   
1619    }
1620    return GetCurrentState();
1621}
1622// --------------------------------------------------------------------------
1623//
1624//! set the opened files service on or off.
1625//! @param evt
1626//!        the current event. contains the instruction string. similar to setdebugonoff
1627//! @returns
1628//!        the new state. Which, in that case, is the current state
1629//!
1630int DataLogger::SetOpenedFilesOnOff(const Event& evt)
1631{
1632    bool backupOpened = fOpenedFilesIsOn;
1633    fOpenedFilesIsOn = evt.GetBool();
1634    if (fOpenedFilesIsOn == backupOpened)
1635        Warn("Warning: opened files service mode was already in the requested state");
1636    else
1637    {
1638        ostringstream str;
1639        str << "Opened files service mode is now " << fOpenedFilesIsOn;
1640        Message(str);
1641    }
1642    return GetCurrentState();
1643   
1644}
1645// --------------------------------------------------------------------------
1646//
1647//! set the number of subscriptions and opened fits on and off
1648//! @param evt
1649//!        the current event. contains the instruction string. similar to setdebugonoff
1650//! @returns
1651//!        the new state. Which, in that case, is the current state
1652//!
1653int DataLogger::SetNumSubsAndFitsOnOff(const Event& evt)
1654{
1655    bool backupSubs = fNumSubAndFitsIsOn;
1656    fNumSubAndFitsIsOn = evt.GetBool();
1657    if (fNumSubAndFitsIsOn == backupSubs)
1658        Warn("Warning: Number of subscriptions service mode was already in the requested state");
1659    else
1660    {
1661        ostringstream str;
1662        str << "Number of subscriptions service mode is now " << fNumSubAndFitsIsOn;
1663        Message(str);
1664    }
1665    return GetCurrentState();
1666}
1667// --------------------------------------------------------------------------
1668//
1669//!    Sets the path to use for the Nightly log file.
1670//! @param evt
1671//!     the event transporting the path
1672//! @returns
1673//!        currently only the current state.
1674//
1675int DataLogger::ConfigureNightlyFileName(const Event& evt)
1676{
1677    if (evt.GetText() != NULL)
1678    {
1679        string givenPath = string(evt.GetText());
1680        if (!DoesPathExist(givenPath))
1681        {
1682            Error("Provided path is not a valid folder. Ignored");
1683            return GetCurrentState();
1684        }
1685        fNightlyFilePath = givenPath;
1686        Message("New Nightly folder specified: " + fNightlyFilePath);
1687    }
1688    else
1689        Error("Empty Nightly folder given. Please specify a valid path.");
1690
1691    return GetCurrentState();
1692}
1693// --------------------------------------------------------------------------
1694//
1695//! Sets the path to use for the run log file.
1696//! @param evt
1697//!        the event transporting the path
1698//! @returns
1699//!     currently only the current state
1700int DataLogger::ConfigureRunFileName(const Event& evt)
1701{
1702    if (evt.GetText() != NULL)
1703    {
1704        string givenPath = string(evt.GetText());
1705        if (!DoesPathExist(givenPath))
1706        {
1707            Error("Provided path is not a valid folder. Ignored");
1708            return GetCurrentState();
1709        }
1710        fRunFilePath = givenPath;
1711        Message("New Run folder specified: " + fRunFilePath);
1712    }
1713    else
1714        Error("Empty Nightly folder given. Please specify a valid path");
1715
1716    return GetCurrentState();
1717}
1718// --------------------------------------------------------------------------
1719//
1720//! Sets the run number.
1721//! @param evt
1722//!        the event transporting the run number
1723//! @returns
1724//!     currently only the current state
1725int DataLogger::ConfigureRunNumber(const Event& evt)
1726{
1727    AddNewRunNumber(evt.GetXtra(), evt.GetTime());
1728    return GetCurrentState();
1729}
1730// --------------------------------------------------------------------------
1731//
1732//! Notifies the DIM service that a particular file was opened
1733//! @ param name the base name of the opened file, i.e. without path nor extension.
1734//!     WARNING: use string instead of string& because I pass values that do not convert to string&.
1735//!        this is not a problem though because file are not opened so often.
1736//! @ param type the type of the opened file. 0 = none open, 1 = log, 2 = text, 4 = fits
1737inline void DataLogger::NotifyOpenedFile(const string &name, int type, DimDescribedService* service)
1738{
1739    if (!fOpenedFilesIsOn)
1740        return;
1741
1742    if (fDebugIsOn)
1743    {
1744        ostringstream str;
1745        str << "Updating " << service->getName() << " file '" << name << "' (type=" << type << ")";
1746        Debug(str);
1747
1748        str.str("");
1749        str << "Num subs: " << fNumSubAndFitsData.numSubscriptions << " Num open FITS: " << fNumSubAndFitsData.numOpenFits;
1750        Debug(str);
1751    }
1752
1753    if (name.size()+1 > FILENAME_MAX)
1754    {
1755        Error("Provided file name '" + name + "' is longer than allowed file name length");
1756    }
1757
1758    OpenFileToDim fToDim;
1759    fToDim.code = type;
1760    memcpy(fToDim.fileName, name.c_str(), name.size()+1);
1761
1762    service->setData(reinterpret_cast<void*>(&fToDim), name.size()+1+sizeof(int));
1763    service->setQuality(0);
1764    service->updateService();
1765}
1766// --------------------------------------------------------------------------
1767//
1768//! Implements the Start transition.
1769//! Concatenates the given path for the Nightly file and the filename itself (based on the day),
1770//! and tries to open it.
1771//! @returns
1772//!        kSM_NightlyOpen if success, kSM_BadNightlyConfig if failure
1773int DataLogger::StartPlease()
1774{
1775    if (fDebugIsOn)
1776    {
1777        Debug("Starting...");   
1778    }
1779    fFullNightlyLogFileName = CompileFileName(fNightlyFilePath, "", "log");
1780    OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName);
1781
1782    fFullNightlyReportFileName = CompileFileName(fNightlyFilePath, "", "rep");
1783    OpenTextFilePlease(fNightlyReportFile, fFullNightlyReportFileName);
1784
1785    if (!fNightlyLogFile.is_open() || !fNightlyReportFile.is_open())
1786    {   
1787        ostringstream str;
1788        str << "Something went wrong while openning nightly files " << fFullNightlyLogFileName << " and " << fFullNightlyReportFileName;
1789        Error(str);
1790        return kSM_BadNightlyConfig;
1791    }
1792    //get the size of the newly opened file.
1793    fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName);
1794    fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName);
1795    fFileSizesMap.clear();
1796    fBaseSizeRun = 0;
1797    fPreviousSize = 0;
1798
1799    //notify that a new file has been opened.
1800    string baseFileName = CompileFileName(fNightlyFilePath, "", "");
1801    NotifyOpenedFile(baseFileName, 3, fOpenedNightlyFiles);
1802
1803    fOpenedNightlyFits.clear();
1804   
1805    return kSM_NightlyOpen;     
1806}
1807
1808#ifdef HAVE_FITS
1809// --------------------------------------------------------------------------
1810//
1811//! open if required a the FITS files corresponding to a given subscription
1812//! @param sub
1813//!     the current DimInfo subscription being examined
1814void DataLogger::OpenFITSFilesPlease(SubscriptionType& sub, RunNumberType* cRunNumber)
1815{
1816    string serviceName(sub.dimInfo->getName());
1817    //if run number has changed, reopen a new fits file with the correct run number.
1818     if (sub.runFile.IsOpen() && sub.runFile.fRunNumber != sub.runNumber)
1819     {
1820         if (fDebugIsOn)
1821             Debug("Run number changed. Closing " + sub.runFile.fFileName);
1822         sub.runFile.Close();
1823     }
1824      //we must check if we should group this service subscription to a single fits file before we replace the / by _
1825    bool hasGrouping = false;
1826    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
1827    {//will we find this service in the grouping list ?
1828        for (set<string>::const_iterator it=fGrouping.begin(); it!=fGrouping.end(); it++)
1829        {
1830            if (serviceName.find(*it) != string::npos)
1831            {
1832                hasGrouping = true;
1833                break;
1834            }
1835        }
1836    }
1837    hasGrouping = true;
1838    for (unsigned int i=0;i<serviceName.size(); i++)
1839    {
1840        if (serviceName[i] == '/')
1841        {
1842            serviceName[i] = '_';
1843            break;   
1844        }   
1845    }
1846    //we open the NightlyFile anyway, otherwise this function shouldn't have been called.
1847    if (!sub.nightlyFile.IsOpen())
1848    {
1849        string fileNameOnly, partialName;
1850        partialName = CompileFileName(fNightlyFilePath, serviceName, "fits");
1851        fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1852        if (!sub.fitsBufferAllocated)
1853            AllocateFITSBuffers(sub);
1854        //get the size of the file we're about to open
1855        if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update)
1856            fOpenedNightlyFits[fileNameOnly].push_back(serviceName);
1857
1858        if (!sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, 0))
1859        {
1860            SetCurrentState(kSM_WriteError);
1861            return;
1862        }
1863        //notify the opening
1864        string baseFileName = CompileFileName(fNightlyFilePath, "", "");
1865        NotifyOpenedFile(baseFileName, 7, fOpenedNightlyFiles);
1866        if (fNumSubAndFitsIsOn)
1867            fNumSubAndFits->updateService();
1868        if (fDebugIsOn)
1869        {
1870            ostringstream str;
1871            str << "Opened Nightly FITS: " << partialName << " and table: " << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
1872            Debug(str);
1873        }
1874    }
1875    //do the actual file open
1876    if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging) && sub.runNumber != 0)
1877    {//buffer for the run file have already been allocated when doing the Nightly file
1878        string fileNameOnly;
1879        string partialName;
1880        if (hasGrouping)
1881        {
1882            partialName = CompileFileName(fRunFilePath, sub.runNumber, "group", "fits");
1883            fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1884        }
1885        else
1886        {
1887            partialName = CompileFileName(fRunFilePath, sub.runNumber, serviceName, "fits");
1888            fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1889        }
1890        //get the size of the file we're about to open
1891        if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update)
1892            cRunNumber->openedFits[fileNameOnly].push_back(serviceName);
1893        else
1894            if (hasGrouping)
1895            {
1896             cRunNumber->addServiceToOpenedFits(fileNameOnly, serviceName);
1897            }
1898
1899        if (hasGrouping && (!cRunNumber->runFitsFile.get()))
1900            try
1901            {
1902                cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>(new CCfits::FITS(partialName, CCfits::RWmode::Write));
1903                (fNumSubAndFitsData.numOpenFits)++;
1904            }   
1905            catch (CCfits::FitsException e)
1906            {
1907                ostringstream str;
1908                str << "Could not open FITS Run file " << partialName << " reason: " << e.message();
1909                Error(str);
1910                cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>();//NULL;
1911            }
1912
1913        string baseFileName = CompileFileName(fRunFilePath, sub.runNumber, "", "");
1914        NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles);// + '_' + serviceName, 4);
1915        if (hasGrouping)
1916        {
1917            if (!sub.runFile.Open(partialName, serviceName, (cRunNumber->runFitsFile).get(), &fNumSubAndFitsData.numOpenFits, this, sub.runNumber))
1918            {
1919                SetCurrentState(kSM_WriteError);
1920                return;
1921            }
1922        }
1923        else
1924        {
1925            if (sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber))
1926            {
1927                SetCurrentState(kSM_WriteError);
1928                return;
1929            }
1930        }
1931       if (fNumSubAndFitsIsOn)
1932           fNumSubAndFits->updateService();
1933           if (fDebugIsOn)
1934        {
1935            ostringstream str;
1936            str << "Opened Run FITS: " << partialName << " and table: " << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits;
1937            Debug(str);
1938        }
1939    }
1940}   
1941// --------------------------------------------------------------------------
1942//
1943//! Allocates the required memory for a given pair of fits files (nightly and run)
1944//! @param sub the subscription of interest.
1945//
1946void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
1947{
1948    int size = sub.dimInfo->getSize();
1949     
1950    //Init the time columns of the file
1951    Description dateDesc(string("Time"), string("Modified Julian Date"), string("MjD"));
1952    sub.nightlyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1953    sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1954
1955    Description QoSDesc("Qos", "Quality of service", "None");
1956    sub.nightlyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1957    sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1958
1959    const Converter::FormatList flist = sub.fConv->GetList();
1960    // Compilation failed
1961    if (!sub.fConv->valid())
1962    {
1963        Error("Compilation of format string failed.");
1964        return;
1965    }
1966
1967    //we've got a nice structure describing the format of this service's messages.
1968    //Let's create the appropriate FITS columns
1969    vector<string> dataFormatsLocal;
1970    for (unsigned int i=0;i<flist.size()-1;i++)
1971    {
1972         ostringstream dataQualifier;
1973
1974         dataQualifier << flist[i].second.first;
1975         switch (flist[i].first.first->name()[0])
1976         {
1977             case 'c':
1978             case 'C':
1979                 dataQualifier.str("S");
1980             break;
1981             case 's':
1982                 dataQualifier << "I";
1983             break;
1984             case 'i':
1985             case 'I':
1986                 dataQualifier << "J";
1987             break;
1988             case 'l':
1989             case 'L':
1990                 dataQualifier << "J";
1991             break;
1992             case 'f':
1993             case 'F':
1994                 dataQualifier << "E";
1995             break;
1996             case 'd':
1997             case 'D':
1998                 dataQualifier << "D";
1999             break;
2000             case 'x':
2001             case 'X':
2002                 dataQualifier << "K";
2003             break;
2004             case 'S':
2005                 //for strings, the number of elements I get is wrong. Correct it
2006                 dataQualifier.str(""); //clear
2007                 dataQualifier << size-1 <<  "A";
2008                 size = size-1;
2009             break;
2010             
2011             default:
2012                 Fatal("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 1198.");
2013         };
2014         //we skip the variable length strings for now (in fits only)
2015         if (dataQualifier.str() != "S")
2016             dataFormatsLocal.push_back(dataQualifier.str());
2017     }
2018     sub.nightlyFile.InitDataColumns(GetDescription(sub.server, sub.service), dataFormatsLocal, sub.dimInfo->getData(), size);
2019     sub.runFile.InitDataColumns(GetDescription(sub.server, sub.service), dataFormatsLocal, sub.dimInfo->getData(), size);
2020     sub.fitsBufferAllocated = true;
2021}
2022// --------------------------------------------------------------------------
2023//
2024//! write a dimInfo data to its corresponding FITS files
2025//
2026void DataLogger::WriteToFITS(SubscriptionType& sub)
2027{
2028        //nightly File status (open or not) already checked
2029        if (sub.nightlyFile.IsOpen())
2030        {
2031            if (!sub.nightlyFile.Write(sub.fConv.get()))
2032                SetCurrentState(kSM_WriteError);
2033         }
2034        if (sub.runFile.IsOpen())
2035        {
2036            if (!sub.runFile.Write(sub.fConv.get()))
2037                SetCurrentState(kSM_WriteError);
2038        }
2039}
2040#endif //if has_fits
2041
2042std::string DataLogger::SetCurrentState(int state, const char *txt, const std::string &cmd)
2043{
2044    if (state == kSM_WriteError && GetCurrentState() == kSM_WriteError)
2045        return "";
2046    return StateMachineImp::SetCurrentState(state, txt, cmd);
2047}
2048// --------------------------------------------------------------------------
2049//
2050//! Implements the StartRun transition.
2051//! Concatenates the given path for the run file and the filename itself (based on the run number),
2052//! and tries to open it.
2053//! @returns
2054//!        kSM_Logging if success, kSM_BadRunConfig if failure.
2055int DataLogger::StartRunPlease()
2056{
2057    if (fDebugIsOn)
2058    {
2059        Debug("Starting Run Logging...");   
2060    }
2061    //open all the relevant run-files. i.e. all the files associated with run numbers.
2062    for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
2063        OpenRunFile(*it);
2064
2065    return kSM_Logging;
2066}
2067
2068#ifdef HAVE_FITS
2069// --------------------------------------------------------------------------
2070//
2071//! Create a fits group file with all the run-fits that were written (either daily or run)
2072//! @param filesToGroup a map of filenames mapping to table names to be grouped (i.e. a
2073//!        single file can contain several tables to group
2074//! @param runNumber the run number that should be used for grouping. 0 means nightly group
2075//
2076void DataLogger::CreateFitsGrouping(map<string, vector<string> > & filesToGroup, int runNumber)
2077{
2078    if (fDebugIsOn)
2079    {
2080        ostringstream str;
2081        str << "Creating fits group for ";
2082        if (runNumber != 0)
2083            str << "run files";
2084        else
2085            str << "nightly files";
2086        Debug(str);
2087    }
2088    //create the FITS group corresponding to the ending run.
2089    CCfits::FITS* groupFile;
2090    unsigned int numFilesToGroup = 0;
2091    for (map<string, vector<string> >::const_iterator it=filesToGroup.begin(); it != filesToGroup.end(); it++)
2092    {
2093        numFilesToGroup += it->second.size();
2094    }
2095    if (fDebugIsOn)
2096    {
2097        ostringstream str;
2098        str << "There are " << numFilesToGroup << " tables to group";
2099        Debug(str);
2100    }
2101    if (numFilesToGroup <= 1)
2102    {
2103        filesToGroup.clear();
2104        return;
2105    }
2106    string groupName;
2107    if (runNumber != 0)
2108        groupName = CompileFileName(fRunFilePath, runNumber, "", "fits");
2109    else
2110        groupName = CompileFileName(fNightlyFilePath, "", "fits");
2111
2112    CCfits::Table* groupTable;
2113    int maxCharLength = 50;//FILENAME_MAX;
2114    try
2115    {
2116        groupFile = new CCfits::FITS(groupName, CCfits::RWmode::Write);
2117        //setup the column names
2118        ostringstream pathTypeName;
2119        pathTypeName << maxCharLength << "A";
2120        vector<string> names;
2121        vector<string> dataTypes;
2122        names.push_back("MEMBER_XTENSION");
2123        dataTypes.push_back("8A");
2124        names.push_back("MEMBER_URI_TYPE");
2125        dataTypes.push_back("3A");
2126        names.push_back("MEMBER_LOCATION");
2127        dataTypes.push_back(pathTypeName.str());
2128        names.push_back("MEMBER_NAME");
2129        dataTypes.push_back(pathTypeName.str());
2130
2131        groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
2132//TODO handle the case when the logger was stopped and restarted during the same day, i.e. the grouping file must be updated
2133     }
2134     catch (CCfits::FitsException e)
2135     {
2136         ostringstream str;
2137         str << "Could not open or create FITS table GROUPING in  file " << groupName << " reason: " << e.message();
2138         Error(str);
2139         return;
2140     }
2141
2142    //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about.
2143    //use cfitsio routines instead
2144    groupTable->makeThisCurrent();
2145    //create appropriate buffer.
2146    const unsigned int n = 8 + 3 + 2*maxCharLength + 1; //+1 for trailling character
2147
2148    vector<unsigned char> realBuffer;
2149    realBuffer.resize(n);
2150    unsigned char* fitsBuffer = &realBuffer[0];
2151    memset(fitsBuffer, 0, n);
2152
2153    char* startOfExtension = reinterpret_cast<char*>(fitsBuffer);
2154    char* startOfURI       = reinterpret_cast<char*>(&fitsBuffer[8]);
2155    char* startOfLocation  = reinterpret_cast<char*>(&fitsBuffer[8 + 3]);
2156    char* startOfName      = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]);
2157
2158    strcpy(startOfExtension, "BINTABLE");
2159    strcpy(startOfURI,       "URL");
2160
2161    int i=1;
2162    for (map<string, vector<string> >::const_iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++)
2163        for (vector<string>::const_iterator jt=it->second.begin(); jt != it->second.end(); jt++, i++)
2164        {
2165            strcpy(startOfLocation, it->first.c_str());
2166            strcpy(startOfName, jt->c_str());
2167
2168            if (fDebugIsOn)
2169            {
2170                ostringstream str;
2171                str << "Grouping " << it->first << " " << *jt;
2172                Debug(str);
2173            }
2174
2175            int status = 0;
2176            fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status);
2177            if (status)
2178            {
2179                ostringstream str;
2180                str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status;
2181                Error(str);
2182            }
2183        }
2184
2185    filesToGroup.clear();
2186    delete groupFile;
2187}
2188#endif //HAVE_FITS
2189
2190// --------------------------------------------------------------------------
2191//
2192//! Implements the StopRun transition.
2193//! Attempts to close the run file.
2194//! @returns
2195//!        kSM_WaitingRun if success, kSM_FatalError otherwise
2196int DataLogger::StopRunPlease()
2197{
2198
2199    if (fDebugIsOn)
2200    {
2201        Debug("Stopping Run Logging...");   
2202    }
2203    for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
2204    {
2205#ifdef RUN_LOGS
2206        if (!it->logFile->is_open() || !it->reportFile->is_open())
2207#else
2208        if (!it->reportFile->is_open())
2209#endif
2210            return kSM_FatalError;
2211#ifdef RUN_LOGS
2212        it->logFile->close();
2213#endif
2214        it->reportFile->close();
2215    }
2216
2217#ifdef HAVE_FITS
2218    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2219        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2220        {
2221            if (j->second.runFile.IsOpen())
2222                j->second.runFile.Close();
2223        }
2224#endif
2225    NotifyOpenedFile("", 0, fOpenedRunFiles);
2226    if (fNumSubAndFitsIsOn)
2227        fNumSubAndFits->updateService();
2228
2229    while (fRunNumber.size() > 0)
2230    {
2231        RemoveOldestRunNumber();
2232    }
2233
2234    return kSM_WaitingRun;
2235}
2236// --------------------------------------------------------------------------
2237//
2238//! Implements the Stop and Reset transitions.
2239//! Attempts to close any openned file.
2240//! @returns
2241//!     kSM_Ready
2242int DataLogger::GoToReadyPlease()
2243{
2244   if (fDebugIsOn)
2245   {
2246        Debug("Going to the Ready state...");
2247   }
2248   if (GetCurrentState() == kSM_Logging)
2249       StopRunPlease();
2250
2251    if (fNightlyLogFile.is_open())
2252        fNightlyLogFile.close();
2253    if (fNightlyReportFile.is_open())
2254        fNightlyReportFile.close();
2255       
2256#ifdef HAVE_FITS
2257    for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2258        for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2259        {
2260            if (j->second.nightlyFile.IsOpen())
2261                j->second.nightlyFile.Close();;
2262        }
2263#endif
2264    if (GetCurrentState() == kSM_Logging || 
2265        GetCurrentState() == kSM_WaitingRun || 
2266        GetCurrentState() == kSM_NightlyOpen)
2267    { 
2268        NotifyOpenedFile("", 0, fOpenedNightlyFiles);
2269        if (fNumSubAndFitsIsOn)
2270            fNumSubAndFits->updateService();
2271    }
2272#ifdef HAVE_FITS
2273    CreateFitsGrouping(fOpenedNightlyFits, 0);
2274#endif
2275    return kSM_Ready;
2276}
2277// --------------------------------------------------------------------------
2278//
2279//! Implements the transition towards kSM_WaitingRun
2280//! Does nothing really.
2281//!    @returns
2282//!        kSM_WaitingRun
2283int DataLogger::NightlyToWaitRunPlease()
2284{
2285    if (fDebugIsOn)
2286    {
2287        Debug("Going to Wait Run Number state...");   
2288    }
2289    return kSM_WaitingRun;   
2290}
2291// --------------------------------------------------------------------------
2292//
2293//! Setup Logger's configuration from a Configuration object
2294//! @param conf the configuration object that should be used
2295//!
2296bool DataLogger::SetConfiguration(Configuration& conf)
2297{
2298    fDebugIsOn = conf.Get<bool>("debug");
2299
2300    //Set the block or allow list
2301    fBlackList.clear();
2302    fWhiteList.clear();
2303
2304    //Adding entries that should ALWAYS be ignored
2305    //fBlackList.insert("DATA_LOGGER/");
2306    fBlackList.insert("/SERVICE_LIST");
2307    fBlackList.insert("DIS_DNS/");
2308
2309    if (conf.Has("block"))
2310    {
2311        const vector<string> vec = conf.Get<vector<string>>("block");
2312
2313        fBlackList.insert(vec.begin(), vec.end());
2314    }
2315
2316    if (conf.Has("allow"))
2317    {
2318        const vector<string> vec = conf.Get<vector<string>>("allow");
2319        fWhiteList.insert(vec.begin(), vec.end());
2320    }
2321
2322    //Set the grouping
2323    if (conf.Has("group"))
2324    {
2325        const vector<string> vec = conf.Get<vector<string>>("group");
2326        fGrouping.insert(vec.begin(), vec.end());
2327    }
2328    return true;
2329}
2330
2331// --------------------------------------------------------------------------
2332int RunDim(Configuration &conf)
2333{
2334    WindowLog wout;
2335
2336    //log.SetWindow(stdscr);
2337    if (conf.Has("log"))
2338        if (!wout.OpenLogFile(conf.Get<string>("log")))
2339            wout << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;
2340
2341    // Start io_service.Run to use the StateMachineImp::Run() loop
2342    // Start io_service.run to only use the commandHandler command detaching
2343    DataLogger logger(wout);
2344    if (!logger.SetConfiguration(conf))
2345        return -1;
2346
2347    logger.Run(true);
2348
2349    return 0;
2350}
2351// --------------------------------------------------------------------------
2352void RunThread(DataLogger* logger)
2353{
2354    // This is necessary so that the StateMachine Thread can signal the
2355    // Readline thread to exit
2356    logger->Run(true);
2357    Readline::Stop();   
2358}
2359// --------------------------------------------------------------------------
2360template<class T>
2361int RunShell(Configuration &conf)
2362{
2363    static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);
2364
2365    WindowLog &win  = shell.GetStreamIn();
2366    WindowLog &wout = shell.GetStreamOut();
2367
2368    if (conf.Has("log"))
2369        if (!wout.OpenLogFile(conf.Get<string>("log")))
2370            win << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;
2371
2372    DataLogger logger(wout);
2373
2374    if (!logger.SetConfiguration(conf))
2375        return -1;
2376   
2377    shell.SetReceiver(logger);
2378
2379    boost::thread t(boost::bind(RunThread, &logger));
2380   
2381    shell.Run(); // Run the shell
2382   
2383    logger.Stop();
2384   
2385    //Wait until the StateMachine has finished its thread
2386    //before returning and destroyinng the dim objects which might
2387    //still be in use.
2388    t.join();
2389
2390    return 0;
2391}
2392
2393/*
2394 Extract usage clause(s) [if any] for SYNOPSIS.
2395 Translators: "Usage" and "or" here are patterns (regular expressions) which
2396 are used to match the usage synopsis in program output.  An example from cp
2397 (GNU coreutils) which contains both strings:
2398  Usage: cp [OPTION]... [-T] SOURCE DEST
2399    or:  cp [OPTION]... SOURCE... DIRECTORY
2400    or:  cp [OPTION]... -t DIRECTORY SOURCE...
2401 */
2402void PrintUsage()
2403{
2404    cout << "\n"
2405        "The data logger connects to all available Dim services and "
2406        "writes them to ascii and fits files.\n"
2407        "\n"
2408        "The default is that the program is started without user interaction. "
2409        "All actions are supposed to arrive as DimCommands. Using the -c "
2410        "option, a local shell can be initialized. With h or help a short "
2411        "help message about the usage can be brought to the screen.\n"
2412        "\n"
2413        "Usage: dataLogger [-c type] [OPTIONS]\n"
2414        "  or:  dataLogger [OPTIONS]\n";
2415    cout << endl;
2416
2417}
2418// --------------------------------------------------------------------------
2419void PrintHelp()
2420{
2421    /* Additional help text which is printed after the configuration
2422     options goes here */
2423    cout <<
2424        "\n"
2425        "If the allow list has any element, only the servers and/or services "
2426        "specified in the list will be used for subscription. The black list "
2427        "will disable service subscription and has higher priority than the "
2428        "allow list. If the allow list is not present by default all services "
2429        "will be subscribed."
2430        "\n"
2431        "For example, block=DIS_DNS/ will skip all the services offered by "
2432        "the DIS_DNS server, while block=/SERVICE_LIST will skip all the "
2433        "SERVICE_LIST services offered by any server and DIS_DNS/SERVICE_LIST "
2434        "will skip DIS_DNS/SERVICE_LIST.\n"
2435        "\n"
2436        "The commands offered by the dataLoger are the following: \n";
2437    cout << setw(20) << DataLogger::fConfigDay << " : specify the path where to put the nightly files\n";
2438    cout << setw(20) << DataLogger::fConfigRun << " : specify the path where to put the run files\n";
2439    cout << setw(20) << DataLogger::fConfigRunNumber << " : specify the run number\n";
2440    cout << setw(20) << DataLogger::fConfigLog << " : log a particular message\n";
2441    cout << setw(20) << DataLogger::fTransStart << " : start the nightly logging\n";
2442    cout << setw(20) << DataLogger::fTransStop << " : stop the nightly logging\n";
2443    cout << setw(20) << DataLogger::fTransStartRun << " : start the run logging\n";
2444    cout << setw(20) << DataLogger::fTransStopRun << " : stop the run logging\n";
2445    cout << setw(20) << DataLogger::fTransReset << " : stop any logging and/or recover from an error state\n";
2446    cout << setw(20) << DataLogger::fTransWait << " : go to the wait for run number state\n";
2447    cout << setw(20) << DataLogger::fPrintCommand << " : print the current state of the logger to the shell\n";
2448    cout << setw(20) << DataLogger::fDebugOnOff << " : turn on or off the debug mode\n";
2449    cout << setw(20) << DataLogger::fStatsPeriod << " : set the periodicity of the statistics. 0 disable them\n";
2450    cout << endl;
2451}
2452// --------------------------------------------------------------------------
2453void SetupConfiguration(Configuration &conf)
2454{
2455    const string n = conf.GetName()+".log";
2456
2457    po::options_description configp("Program options");
2458    configp.add_options()
2459        ("dns",       var<string>("localhost"),  "Dim nameserver host name (Overwites DIM_DNS_NODE environment variable)")
2460        ("log,l",     var<string>(n), "Write log-file")
2461        ("console,c", var<int>(),     "Use console (0=shell, 1=simple buffered, X=simple unbuffered)")
2462        ;
2463
2464    po::options_description configs("DataLogger options");
2465    configs.add_options()
2466        ("block,b", vars<string>(), "Black-list of services")
2467        ("allow,a", vars<string>(), "White-list of services")
2468        ("debug",   po_bool(),      "Debug mode. Print clear text of received service reports to log-stream")
2469        ("group,g", vars<string>(), "Grouping of services into a single run-Fits")
2470        ;
2471
2472    conf.AddEnv("dns", "DIM_DNS_NODE");
2473
2474    conf.AddOptions(configp);
2475    conf.AddOptions(configs);
2476}
2477// --------------------------------------------------------------------------
2478int main(int argc, const char* argv[])
2479{
2480    Configuration conf(argv[0]);
2481    conf.SetPrintUsage(PrintUsage);
2482    SetupConfiguration(conf);
2483
2484    po::variables_map vm;
2485    try
2486    {
2487        vm = conf.Parse(argc, argv);
2488    }
2489#if BOOST_VERSION > 104000
2490    catch (po::multiple_occurrences &e)
2491    {
2492        cerr << "Program options invalid due to: " << e.what() << " of '" << e.get_option_name() << "'." << endl;
2493        return -1;
2494    }
2495#endif
2496    catch (exception& e)
2497    {
2498        cerr << "Program options invalid due to: " << e.what() << endl;
2499        return -1;
2500    }
2501
2502    if (conf.HasVersion() || conf.HasPrint())
2503        return -1;
2504
2505    if (conf.HasHelp())
2506    {
2507        PrintHelp();
2508        return -1;
2509    }
2510
2511    Dim::Setup(conf.Get<string>("dns"));
2512
2513//    try
2514    {
2515        // No console access at all
2516        if (!conf.Has("console"))
2517            return RunDim(conf);
2518
2519        // Console access w/ and w/o Dim
2520        if (conf.Get<int>("console")==0)
2521            return RunShell<LocalShell>(conf);
2522        else
2523            return RunShell<LocalConsole>(conf);
2524    }
2525/*    catch (exception& e)
2526    {
2527        cerr << "Exception: " << e.what() << endl;
2528        return -1;
2529    }*/
2530
2531    return 0;
2532}
Note: See TracBrowser for help on using the repository browser.