source: trunk/FACT++/src/datalogger.cc@ 14088

Last change on this file since 14088 was 14009, checked in by tbretz, 12 years ago
Exit with 127 if parsing of command line options failed.
File size: 87.4 KB
Line 
1//****************************************************************
2/** @class DataLogger
3
4 @brief Logs all message and infos between the services
5
6 This is the main logging class facility.
7 It derives from StateMachineDim and DimInfoHandler. the first parent is here to enforce
8 a state machine behaviour, while the second one is meant to make the dataLogger receive
9 dim services to which it subscribed from.
10 The possible states and transitions of the machine are:
11 \dot
12 // FIXME FIXME: Error states missing...
13 digraph datalogger
14 {
15 node [shape=record, fontname=Helvetica, fontsize=10];
16
17 srt [label="Start" style="rounded"]
18 rdy [label="Ready"]
19 nop [label="NightlyOpen"]
20 wait [label="WaitingRun"]
21 log [label="Logging"]
22
23 //e [label="Error" color="red"];
24 //c [label="BadFolder" color="red"]
25
26
27 cmd_start [label="START" shape="none" height="0"]
28 cmd_stop [label="STOP" shape="none" height="0"]
29 cmd_stopr [label="STOP_RUN_LOGGING" shape="none" height="0"]
30 cmd_startr [label="START_RUN_LOGGING" shape="none" height="0"]
31
32 { rank=same; cmd_startr cmd_stopr }
33 { rank=same; cmd_start cmd_stop }
34
35
36 srt -> rdy
37
38 rdy -> cmd_start [ arrowhead="open" dir="both" arrowtail="tee" weight=10 ]
39 cmd_start -> nop
40
41 nop -> cmd_stop [ arrowhead="none" dir="both" arrowtail="inv" ]
42 wait -> cmd_stop [ arrowhead="none" dir="both" arrowtail="inv" ]
43 log -> cmd_stop [ arrowhead="none" dir="both" arrowtail="inv" ]
44 cmd_stop -> rdy
45
46 wait -> cmd_stopr [ arrowhead="none" dir="both" arrowtail="inv" ]
47 log -> cmd_stopr [ arrowhead="none" dir="both" arrowtail="inv" ]
48 cmd_stopr -> nop
49
50 nop -> cmd_startr [ arrowhead="none" dir="both" arrowtail="inv" weight=10 ]
51 rdy -> cmd_startr [ arrowhead="none" dir="both" arrowtail="inv" ]
52 cmd_startr -> wait [ weight=10 ]
53
54
55 wait -> log
56 log -> wait
57 }
58 \enddot
59
60 For questions or bug report, please contact Etienne Lyard (etienne.lyard@unige.ch) or Thomas Bretz.
61 */
62 //****************************************************************
63#include <unistd.h> //for getting stat of opened files
64//#include <sys/statvfs.h> //for getting disk free space
65//#include <sys/stat.h> //for getting files sizes
66#include <fstream>
67#include <functional>
68
69#include <boost/filesystem.hpp>
70
71#include "Dim.h"
72#include "Event.h"
73#include "StateMachineDim.h"
74#include "LocalControl.h"
75#include "Configuration.h"
76#include "Converter.h"
77#include "DimWriteStatistics.h"
78
79#include "Description.h"
80#include "DimNetwork.h"
81
82
83#ifdef HAVE_FITS
84#include "Fits.h"
85#endif
86
87//Dim structures
88///distributes the number of opened subscriptions and fits files
89struct NumSubAndFitsType {
90 uint32_t numSubscriptions;
91 uint32_t numOpenFits;
92};
93///distributes which files were opened.
94struct OpenFileToDim {
95 uint32_t code;
96 char fileName[FILENAME_MAX];
97};
98
99///Run number record. Used to keep track of which run numbers are still active
100struct RunNumberType {
101#ifdef RUN_LOGS
102 ///the run number log file
103 shared_ptr<ofstream> logFile;
104#endif
105 ///the run number report file
106 shared_ptr<ofstream> reportFile;
107#ifdef HAVE_FITS
108 ///the run number group fits file
109 shared_ptr<CCfits::FITS> runFitsFile;
110#endif
111#ifdef RUN_LOGS
112 ///the log filename
113 string logName;
114#endif
115 ///the report filename
116 string reportName;
117 ///the actual run number
118 int32_t runNumber;
119 ///the time at which the run number was received
120 Time time;
121 ///list of opened fits used to create the fits grouping when the run ends
122 map<string, vector<string> > openedFits;
123 ///default constructor
124 RunNumberType()
125 {
126#ifdef RUN_LOGS
127 logFile = shared_ptr<ofstream>(new ofstream());
128#endif
129 reportFile = shared_ptr<ofstream>(new ofstream());
130#ifdef HAVE_FITS
131 runFitsFile = shared_ptr<CCfits::FITS>();
132#endif
133 runNumber = 0;
134 }
135 ///default destructor
136 ~RunNumberType()
137 {
138
139 }
140
141 void addServiceToOpenedFits(const string& fileName, const string& serviceName)
142 {
143 //most likely I should add this service name.
144 //the only case for which I should not add it is if a service disapeared, hence the file was closed
145 //and reopened again. Unlikely to happen, but well it may
146
147 if (find(openedFits[fileName].begin(), openedFits[fileName].end(),
148 serviceName)==openedFits[fileName].end())
149 openedFits[fileName].push_back(serviceName);
150 }
151};
152///Dim subscription type. Stores all the relevant info to handle a Dim subscription
153struct SubscriptionType
154{
155#ifdef HAVE_FITS
156 ///Nightly FITS output file
157 Fits nightlyFile;
158 ///run-specific FITS output file
159 Fits runFile;
160#endif
161 ///the server
162 string server;
163 ///the service
164 string service;
165 ///the converter for outputting the data according to the format
166 shared_ptr<Converter> fConv;
167 ///the current run number used by this subscription
168 int32_t runNumber;
169 ///time of the latest received event
170 Time lastReceivedEvent;
171 ///whether or not the fits buffer was allocated already
172 bool fitsBufferAllocated;
173
174 ///the actual dimInfo pointer (must be the last in the list to ensure
175 /// that it is the first which is deleted -- and consequently none of
176 /// the other members can still be in use in an infoHandler)
177 shared_ptr<DimStampedInfo> dimInfo;
178
179 ///Dim info constructor
180 SubscriptionType(DimStampedInfo* info=NULL)
181 {
182 fConv = shared_ptr<Converter>();
183 runNumber = 0;
184 lastReceivedEvent = Time::None;
185 fitsBufferAllocated = false;
186
187 // Should be the last instantiated to make sure that all other
188 // variables which might be used are already initialized
189 dimInfo = shared_ptr<DimStampedInfo>(info);
190 }
191
192 ///default destructor
193 ~SubscriptionType()
194 {
195 }
196};
197
198class DataLogger : public StateMachineDim, DimServiceInfoListImp
199{
200public:
201 /// The list of existing states specific to the DataLogger
202 enum
203 {
204 kSM_NightlyOpen = 20, ///< Nightly file openned and writing
205 kSM_WaitingRun = 30, ///< waiting for the run number to open the run file
206 kSM_Logging = 40, ///< both files openned and writing
207 kSM_BadFolder = 0x101, ///< the folder specified for Nightly logging does not exist or has bad permissions
208 kSM_RunWriteError = 0x103, ///< Denotes that an error occured while writing a run file (text or fits).
209 kSM_DailyWriteError = 0x103, ///< Denots that an error occured while writing a daily file (text or fits).
210 } localstates_t;
211
212 DataLogger(ostream &out);
213 ~DataLogger();
214
215 int EvalOptions(Configuration& conf);
216
217private:
218 /************************************************
219 * MEMBER VARIABLES
220 ************************************************/
221 /// ofstream for the NightlyLogfile
222 ofstream fNightlyLogFile;
223 /// ofstream for the Nightly report file
224 ofstream fNightlyReportFile;
225 /// base path of files
226 string fFilePath;
227 ///run numbers
228 list<RunNumberType> fRunNumber;
229 ///old run numbers time-out delay (in seconds)
230 uint32_t fRunNumberTimeout;
231 ///previous run number. to check if changed while logging
232 int fPreviousRunNumber;
233 ///Current Service Quality
234 int fQuality;
235 ///Modified Julian Date
236 double fMjD;
237 ///for obtaining the name of the existing services
238// ServiceList fServiceList;
239 typedef map<const string, map<string, SubscriptionType> > SubscriptionsListType;
240 ///All the services to which we have subscribed to, sorted by server name.
241 SubscriptionsListType fServiceSubscriptions;
242 ///full name of the nightly log file
243 string fFullNightlyLogFileName;
244 ///full name of the nightly report file
245 string fFullNightlyReportFileName;
246 ///variable to track when the statistic were last calculated
247// Time fPreviousStatsUpdateTime;
248 Time fPreviousOldRunNumberCheck;
249 ///boolean to know whether we should close and reopen daily files or not
250 bool fDailyFileDayChangedAlready;
251
252 DimWriteStatistics fFilesStats;
253private:
254 /***************************************************
255 * DIM INFO HANDLER
256 ***************************************************/
257 //overloading of DIM's infoHandler function
258 void infoHandler();
259
260 /***************************************************
261 * TRANSITION FUNCTIONS
262 ***************************************************/
263 ///Reporting method for the services info received
264 void Report(DimInfo* I, SubscriptionType& sub);
265
266 ///Configuration of the nightly file path
267 int ConfigureFilePath(const Event& evt);
268 ///print the current state of the dataLogger
269 int PrintState(const Event& evt);
270 ///checks whether or not the current info being treated is a run number
271 void CheckForRunNumber(DimInfo* I);
272 /// start transition
273 int Start();
274 ///from waiting to logging transition
275 //int StartRun();
276 // from logging to waiting transition
277 int StopRunLogging();
278 ///stop and reset transition
279 int GoToReady();
280 ///from NightlyOpen to waiting transition
281 int NightlyToWaitRun();
282 ///from wait for run number to nightly open
283 int BackToNightlyOpen();
284#ifdef HAVE_FITS
285 ///Open fits files
286 void OpenFITSFiles(SubscriptionType& sub, RunNumberType* cRunNumber);
287 ///Write data to FITS files
288 void WriteToFITS(SubscriptionType& sub);
289 ///Allocate the buffers required for fits
290 void AllocateFITSBuffers(SubscriptionType& sub);
291#endif//has_fits
292
293 /***************************************
294 * DIM SERVICES PROVIDED BY THE DATA LOGGER
295 ***************************************/
296 ///monitoring notification loop
297 void ServicesMonitoring();
298 inline void NotifyOpenedFile(const string &name, int type, DimDescribedService* service);
299 ///Service for opened files
300 DimDescribedService* fOpenedNightlyFiles;
301 DimDescribedService* fOpenedRunFiles;
302 DimDescribedService* fNumSubAndFits;
303 NumSubAndFitsType fNumSubAndFitsData;
304
305 /***************************************************
306 * DATA LOGGER's CONFIGURATION STUFF
307 ***************************************************/
308 ///black/white listing
309 set<string> fBlackList;
310 set<string> fWhiteList;
311 ///list of services to be grouped
312 set<string> fGrouping;
313 ///configuration flags
314 bool fDebugIsOn;
315// float fStatsPeriodDuration;
316 bool fOpenedFilesIsOn;
317 bool fNumSubAndFitsIsOn;
318 //functions for controlling the services behavior
319 int SetDebugOnOff(const Event& evt);
320 int SetStatsPeriod(const Event& evt);
321 int SetOpenedFilesOnOff(const Event& evt);
322 int SetNumSubsAndFitsOnOff(const Event& evt);
323 int SetRunTimeoutDelay(const Event& evt);
324
325 ///boolean to prevent DIM update while desctructing the dataLogger
326 bool fDestructing;
327 /***************************************************
328 * UTILITIES
329 ***************************************************/
330 ///vectors to keep track of opened Fits files, for grouping purposes.
331 map<string, vector<string> > fOpenedNightlyFits;
332 ///creates a group fits file based on a list of files to be grouped
333 void CreateFitsGrouping(map<string, vector<string> >& filesToGroup, int runNumber);
334
335 bool OpenStreamImp(ofstream &stream, const string &filename, bool mightbeopen);
336 bool OpenStream(shared_ptr<ofstream> stream, const string &filename);
337 ///Open the relevant text files related to a particular run
338 int OpenRunFile(RunNumberType& run);
339 ///add a new run number
340 void AddNewRunNumber(int64_t newRun, Time time);
341 std::vector<int64_t> previousRunNumbers;
342 ///removes the oldest run number, and close the relevant files.
343 void RemoveOldestRunNumber();
344 ///retrieves the size of a file
345 off_t GetFileSize(const string&);
346 ///Get the digits of year, month and day for filenames and paths
347 void GetYearMonthDayForFiles(unsigned short& year, unsigned short& month, unsigned short& day);
348 ///Appends the relevant year month day to a given path
349 void AppendYearMonthDaytoPath(string& path);
350 ///Form the files path
351 string CompileFileNameWithPath(const string &path, const string &service, const string & extension, uint32_t run=0/*, const Time &time=Time()*/);
352 ///Form the file names only
353 string CompileFileName(const string& service, const string& extension, uint32_t run=0, const Time& time=Time()) const;
354 ///Check whether service is in black and/or white list
355 bool ShouldSubscribe(const string& server, const string& service);
356 ///Subscribe to a given server and service
357 DimStampedInfo* SubscribeTo(const string& server, const string& service);
358 ///Open a text file and checks for ofstream status
359 bool OpenTextFile(ofstream& stream, const string& name);
360 ///Checks if the input osftream is in error state, and if so close it.
361 bool CheckForOfstreamError(ofstream& out, bool isDailyStream);
362 ///Goes to Write error states
363 void GoToRunWriteErrorState();
364 void GoToNightlyWriteErrorState();
365 ///Checks if a given path exist
366 bool DoesPathExist(string path);
367 ///Check if old run numbers can be trimmed, and if so, do it
368 void TrimOldRunNumbers();
369 ///Create a given directory
370 bool CreateDirectory(string path);
371 /***************************************************
372 * INHERITED FROM DimServiceInfoList
373 ***************************************************/
374 ///Add a new service subscription
375 void AddService(const string&, const string&, const string&, bool);
376 ///Remove a given service subscription
377 void RemoveService(const string, const string, bool);
378 ///Remove all the services associated with a given server
379 void RemoveAllServices(const string&);
380 ///pointer to the dim's subscription that should distribute the run numbers.
381 DimInfo* fRunNumberService;
382 /***************************************************
383 * Overwritten from MessageImp
384 ***************************************************/
385 vector<string> backLogBuffer;
386 bool shouldBackLog;
387 bool fShouldAutoStart;
388 bool fAutoStarted;
389
390 //Current day variable. Used to close nightly files when night changes
391 int fCurrentDay;
392 Time lastFlush;
393public:
394 int Write(const Time &time, const std::string &txt, int qos=kMessage);
395
396}; //DataLogger
397
398// --------------------------------------------------------------------------
399//
400//! Overwritten write function. This way we directly log the datalogger's messages, without going through dim's dns,
401//! thus increasing robustness.
402//! @param time: see MessageImp class param
403//! @param txt: see MessageImp class param
404//! @param qos: see MessageImp class param
405//! @return see MessageImp class param
406//
407int DataLogger::Write(const Time&time, const std::string& txt, int qos)
408{
409 ostringstream ss;
410 ss << "datalogger: " << txt;
411 if (fNightlyLogFile.is_open())
412 {
413 MessageImp mimp(fNightlyLogFile);
414 mimp.Write(time, ss.str(), qos);
415 }
416 else if (shouldBackLog)
417 {
418 ostringstream str;
419 MessageImp mimp(str);
420 mimp.Write(time, ss.str(), qos);
421 backLogBuffer.push_back(str.str());
422 }
423 return StateMachineDim::Write(time, ss.str(), qos);
424}
425// --------------------------------------------------------------------------
426//
427//! Check if a given path exists
428//! @param path the path to be checked
429//! @return whether or not the creation has been successfull
430//
431bool DataLogger::CreateDirectory(string path)
432{
433 try
434 {
435 DimWriteStatistics::CreateDirectory(path);
436 return true;
437 }
438 catch (const runtime_error &e)
439 {
440 Error(e.what());
441 return false;
442 }
443}
444// --------------------------------------------------------------------------
445//
446//! Check if a given path exists
447//! @param path the path to be checked
448//! @return whether or not the given path exists
449//
450bool DataLogger::DoesPathExist(string path)
451{
452 return DimWriteStatistics::DoesPathExist(path, *this);
453}
454
455// --------------------------------------------------------------------------
456//
457//! Add a new service subscription
458//! @param server the server for which the subscription should be created
459//! @param service the service for which the subscription should be created
460//! @param isCmd whether this is a Dim Command or not. Commands are not logged
461//
462void DataLogger::AddService(const string& server, const string& service, const string&, bool isCmd)
463{
464 //dataLogger does not subscribe to commands
465 if (isCmd)
466 return;
467
468 Info("Got request to add service: "+server+"/"+service);
469
470 //check the given subscription against black and white lists
471 if (!ShouldSubscribe(server, service))
472 return;
473
474 map<string, SubscriptionType> &list = fServiceSubscriptions[server];
475
476 if (list.find(service) != list.end())
477 {
478 Error("Service " + server + "/" + service + " is already in the dataLogger's list... ignoring update.");
479 return;
480 }
481
482 list[service].dimInfo.reset(SubscribeTo(server, service));
483 list[service].server = server;
484 list[service].service = service;
485 fNumSubAndFitsData.numSubscriptions++;
486 //check if this is the run numbers service
487 if ((server == "FAD_CONTROL") && (service == "START_RUN"))
488 fRunNumberService = list[service].dimInfo.get();
489 Info("Added subscription to " + server + "/" + service);
490}
491// --------------------------------------------------------------------------
492//
493//! Remove a given service subscription
494//! @param server the server for which the subscription should be removed
495//! @param service the service that should be removed
496//! @param isCmd whether or not this is a command
497//
498void DataLogger::RemoveService(string server, string service, bool isCmd)
499{
500 Info("Got request for removing service: "+server+"/"+service);
501 if (fDestructing)//this function is called by the super class, after the destructor has deleted its own subscriptions
502 return;
503
504 if (isCmd)
505 return;
506
507 if (fServiceSubscriptions.find(server) == fServiceSubscriptions.end())
508 {
509 Error("Request to remove service "+service+" from server "+server+", but service not found.");
510 return;
511 }
512
513 if (fServiceSubscriptions[server].erase(service) != 1)
514 {
515 //check the given subscription against black and white lists
516 if (!ShouldSubscribe(server, service))
517 return;
518
519 Error("Subscription "+server+"/"+service+" could not be removed as it is not present");
520 return;
521 }
522 fNumSubAndFitsData.numSubscriptions--;
523
524 if ((server == "FAD_CONTROL") && (service == "START_RUN"))
525 fRunNumberService = NULL;
526
527 Info("Removed subscription to " + server + "/" + service);
528}
529// --------------------------------------------------------------------------
530//
531//! Remove all the services associated with a given server
532//! @param server the server for which all the services should be removed
533//
534void DataLogger::RemoveAllServices(const string& server)
535{
536 Info("Got request for removing all services from: "+server);
537 if (fServiceSubscriptions.find(server)==fServiceSubscriptions.end())
538 {
539 Warn("Request to remove all services, but corresponding server " + server + " not found.");
540 return;
541 }
542
543 fNumSubAndFitsData.numSubscriptions -= fServiceSubscriptions[server].size();
544
545 fServiceSubscriptions[server].clear();
546 fServiceSubscriptions.erase(server);
547
548 if (server == "FAD_CONTROL")
549 fRunNumberService = NULL;
550
551 if (fDebugIsOn)
552 Debug("Removed all subscriptions to " + server + "/");
553}
554
555// --------------------------------------------------------------------------
556//
557//! Checks if the given ofstream is in error state and if so, close it
558//! @param out the ofstream that should be checked
559//
560bool DataLogger::CheckForOfstreamError(ofstream& out, bool isDailyStream)
561{
562 if (out.good())
563 return true;
564
565 Error("An error occured while writing to a text file. Closing it");
566 if (out.is_open())
567 out.close();
568 if (isDailyStream)
569 GoToNightlyWriteErrorState();
570 else
571 GoToRunWriteErrorState();
572
573 return false;
574}
575
576bool DataLogger::OpenStreamImp(ofstream &stream, const string &filename, bool mightbeopen)
577{
578 if (stream.is_open())
579 {
580 if (!mightbeopen)
581 Error(filename+" was already open when trying to open it.");
582 return mightbeopen;
583 }
584
585 errno = 0;
586 stream.open(filename.c_str(), ios_base::out | ios_base::app);
587 if (!stream /*|| errno!=0*/)
588 {
589 ostringstream str;
590 str << "ofstream::open() failed for '" << filename << "': " << strerror(errno) << " [errno=" << errno << "]";
591 Error(str);
592 return false;
593 }
594
595 if (!stream.is_open())
596 {
597 Error("File "+filename+" not open as it ought to be.");
598 return false;
599 }
600
601 Info("Opened: "+filename);
602
603 return true;
604}
605
606bool DataLogger::OpenStream(shared_ptr<ofstream> stream, const string &filename)
607{
608 return OpenStreamImp(*stream, filename, false);
609}
610
611// --------------------------------------------------------------------------
612//
613//! Open a text file and checks for error code
614//! @param stream the ofstream for which the file should be opened
615//! @name the file name
616//
617bool DataLogger::OpenTextFile(ofstream& stream, const string& name)
618{
619 return OpenStreamImp(stream, name, true);
620}
621
622// --------------------------------------------------------------------------
623//
624//! Create a new dim subscription to a given server and service
625//! @param server the server name
626//! @param service the service name
627//
628DimStampedInfo* DataLogger::SubscribeTo(const string& server, const string& service)
629{
630 if (fDebugIsOn)
631 Debug("Subscribing to service "+server+"/"+service);
632
633 return new DimStampedInfo((server + "/" + service).c_str(), (void*)NULL, 0, this);
634}
635// --------------------------------------------------------------------------
636//
637//! Check whether a service should be subscribed to, based on the black/white list entries
638//! @param server the server name associated with the service being checked
639//! @param service the service name associated with the service being checked
640//
641bool DataLogger::ShouldSubscribe(const string& server, const string& service)
642{
643 if ((fBlackList.find(server + "/") != fBlackList.end()) ||
644 (fBlackList.find(server + "/" + service) != fBlackList.end()) ||
645 (fBlackList.find("/" + service) != fBlackList.end()))
646 {
647 if (fWhiteList.size()>0 &&
648 (fWhiteList.find(server + "/" + service) != fWhiteList.end()))
649 {
650 if (fDebugIsOn)
651 Debug("White list saved service " + server + "/" + service + " from blacklisting");
652 return true;
653 }
654 if (fDebugIsOn)
655 Debug("Blacklist banned service " + server + "/" + service);
656 return false;
657 }
658 return true;
659}
660// --------------------------------------------------------------------------
661//
662//! Compiles a file name
663//! @param path the base path where to put the file
664//! @param time the time at which the file is created
665//! @param service the service name, if any
666//! @param extension the extension to add, if any
667//
668//string DataLogger::CompileFileName(const string &path, const string &service, const string & extension, const Time &time)
669string DataLogger::CompileFileName(const string& service, const string& extension, uint32_t run, const Time& time) const
670{
671 ostringstream str;
672
673 const Time ftime(time);//removed this as already done by nightAsInt: -boost::posix_time::hours(12));
674 str << ftime.NightAsInt();
675
676 if (run>0)
677 str << '_' << setfill('0') << setw(3) << run;
678
679 if (!service.empty())
680 str << '.' << service;
681
682 if (!extension.empty())
683 str << "." << extension;
684
685 return str.str();
686}
687
688string DataLogger::CompileFileNameWithPath(const string& path, const string& service, const string& extension, uint32_t run/*, const Time& time*/)
689{
690 ostringstream str;
691
692 const Time time;
693
694 //calculate time suitable for naming files.
695 const Time ftime = time-boost::posix_time::hours(12);
696
697 //output it
698 str << path << ftime.GetAsStr("/%Y/%m/%d");
699
700 //check if target directory exist
701 if (!DoesPathExist(str.str()))
702 CreateDirectory(str.str());
703
704 str << '/' << CompileFileName(service, extension, run, time);
705
706 return str.str();
707
708
709}
710
711// --------------------------------------------------------------------------
712//
713//!retrieves the size on disk of a file
714//! @param fileName the full file name for which the size on disk should be retrieved
715//! @return the size of the file on disk, in bytes. 0 if the file does not exist or if an error occured
716//
717off_t DataLogger::GetFileSize(const string& fileName)
718{
719 return DimWriteStatistics::GetFileSizeOnDisk(fileName, *this);
720}
721
722// --------------------------------------------------------------------------
723//
724//! Removes the oldest run number and closes the fits files that should be closed
725//! Also creates the fits grouping file
726//
727void DataLogger::RemoveOldestRunNumber()
728{
729 if (fDebugIsOn)
730 {
731 ostringstream str;
732 str << "Removing run number " << fRunNumber.front().runNumber;
733 Debug(str);
734 }
735 CreateFitsGrouping(fRunNumber.front().openedFits, fRunNumber.front().runNumber);
736
737 //crawl through the subscriptions to see if there are still corresponding fits files opened.
738 for (SubscriptionsListType::iterator x=fServiceSubscriptions.begin();
739 x!=fServiceSubscriptions.end(); x++)
740 for (map<string, SubscriptionType>::iterator y=x->second.begin();
741 y!=x->second.end(); y++)
742 if (y->second.runFile.fRunNumber == fRunNumber.front().runNumber && y->second.runFile.IsOpen())
743 {
744 y->second.runFile.Close();
745 }
746 //if a grouping file is on, decrease the number of opened fits manually
747 if (fRunNumber.front().runFitsFile)
748 (fNumSubAndFitsData.numOpenFits)--;
749 //remove the entry
750 fRunNumber.pop_front();
751}
752
753// --------------------------------------------------------------------------
754//
755//! Default constructor. The name of the machine is given DATA_LOGGER
756//! and the state is set to kSM_Ready at the end of the function.
757//
758//!Setup the allows states, configs and transitions for the data logger
759//
760DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER"),
761 fFilesStats("DATA_LOGGER", *this)
762{
763 shouldBackLog = true;
764 //initialize member data
765 fFilePath = ".";
766
767 //calculate time "centered" around noon instead of midnight
768 const Time timeNow;
769 const Time nowMinusTwelve = timeNow-boost::posix_time::hours(12);
770 fCurrentDay = nowMinusTwelve.M()*31 + nowMinusTwelve.D();//assume 31 days per month. we do not really care, only want unique number per day of the year
771 lastFlush = Time();
772
773 //Give a name to this machine's specific states
774 AddStateName(kSM_NightlyOpen, "NightlyFileOpen", "The summary files for the night are open.");
775 AddStateName(kSM_WaitingRun, "WaitForRun", "The summary files for the night are open and we wait for a run to be started.");
776 AddStateName(kSM_Logging, "Logging", "The summary files for the night and the files for a single run are open.");
777 AddStateName(kSM_BadFolder, "ErrInvalidFolder", "The folder for the files is not invalid.");
778 AddStateName(kSM_DailyWriteError, "ErrDailyWrite", "An error occured while writing to a daily (and run) file.");
779 AddStateName(kSM_RunWriteError, "ErrRunWrite", "An error occured while writing to a run file.");
780
781 // Add the possible transitions for this machine
782 AddEvent("START", kSM_Ready, kSM_BadFolder)
783 (bind(&DataLogger::Start, this))
784 ("Start the nightly logging. Nightly file location must be specified already");
785
786 AddEvent("STOP", kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging, kSM_DailyWriteError, kSM_RunWriteError)
787 (bind(&DataLogger::GoToReady, this))
788 ("Stop all data logging, close all files.");
789/*
790 AddEvent(kSM_Logging, "START_RUN", kSM_WaitingRun, kSM_BadRunConfig)
791 (bind(&DataLogger::StartRun, this))
792 ("Start the run logging. Run file location must be specified already.");
793
794 AddEvent(kSM_WaitingRun, "STOP_RUN", kSM_Logging)
795 (bind(&DataLogger::StopRunLogging, this))
796 ("Wait for a run to be started, open run-files as soon as a run number arrives.");
797*/
798 AddEvent("RESET", kSM_Error, kSM_BadFolder, kSM_DailyWriteError, kSM_RunWriteError)
799 (bind(&DataLogger::GoToReady, this))
800 ("Transition to exit error states. Closes the any open file.");
801
802 AddEvent("START_RUN_LOGGING", /*kSM_Logging,*/ kSM_NightlyOpen, kSM_Ready)
803 (bind(&DataLogger::NightlyToWaitRun, this))
804 ("Go to waiting for run number state. In this state with any received run-number a new file is opened.");
805
806 AddEvent("STOP_RUN_LOGGING", kSM_WaitingRun, kSM_Logging)
807 (bind(&DataLogger::BackToNightlyOpen, this))
808 ("Go from the wait for run to nightly open state.");
809
810 /*
811 // Add the possible configurations for this machine
812 AddEvent("SET_FOLDER", "C", kSM_Ready, kSM_BadNightlyConfig)
813 (bind(&DataLogger::ConfigureNightlyFileName, this, placeholders::_1))
814 ("Configure the base folder for the nightly files."
815 "|Path[string]:Absolute or relative path name where the nightly files should be stored.");
816
817 AddEvent("SET_RUN_FOLDER", "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig)
818 (bind(&DataLogger::ConfigureRunFileName, this, placeholders::_1))
819 ("Configure the base folder for the run files."
820 "|Path[string]:Absolute or relative path name where the run files should be stored.");
821 */
822 // Provide a print command
823 AddEvent("PRINT_INFO")
824 (bind(&DataLogger::PrintState, this, placeholders::_1))
825 ("Print information about the internal status of the data logger.");
826
827 OpenFileToDim fToDim;
828 fToDim.code = 0;
829 fToDim.fileName[0] = '\0';
830
831 fOpenedNightlyFiles = new DimDescribedService(GetName() + "/FILENAME_NIGHTLY", "I:1;C", fToDim,
832 "Path and base name used for the nightly files."
833 "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
834 "|Name[string]:path and base file name");
835
836 fOpenedRunFiles = new DimDescribedService(GetName() + "/FILENAME_RUN", "I:1;C", fToDim,
837 "Path and base name used for the run files."
838 "|Type[int]:type of open files (1=log, 2=rep, 4=fits)"
839 "|Name[string]:path and base file name");
840
841 fNumSubAndFitsData.numSubscriptions = 0;
842 fNumSubAndFitsData.numOpenFits = 0;
843 fNumSubAndFits = new DimDescribedService(GetName() + "/NUM_SUBS", "I:2", fNumSubAndFitsData,
844 "Num. open files + num. subscribed services"
845 "|NSubAndOpenFiles[int]:Num. of subs and open files");
846
847 //services parameters
848 fDebugIsOn = false;
849 fOpenedFilesIsOn = true;
850 fNumSubAndFitsIsOn = true;
851
852 // provide services control commands
853 AddEvent("SET_DEBUG_MODE", "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
854 (bind(&DataLogger::SetDebugOnOff, this, placeholders::_1))
855 ("Switch debug mode on or off. Debug mode prints information about every service written to a file."
856 "|Enable[bool]:Enable of disable debug mode (yes/no).");
857
858 AddEvent("SET_STATISTICS_UPDATE_INTERVAL", "S:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
859 (bind(&DataLogger::SetStatsPeriod, this, placeholders::_1))
860 ("Interval in which the data-logger statistics service (STATS) is updated."
861 "|Interval[ms]:Value in milliseconds (<=0: no update).");
862
863 AddEvent("ENABLE_FILENAME_SERVICES", "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
864 (bind(&DataLogger::SetOpenedFilesOnOff ,this, placeholders::_1))
865 ("Switch service which distributes information about the open files on or off."
866 "|Enable[bool]:Enable of disable filename services (yes/no).");
867
868 AddEvent("ENABLE_NUMSUBS_SERVICE", "B:1", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready)
869 (bind(&DataLogger::SetNumSubsAndFitsOnOff, this, placeholders::_1))
870 ("Switch the service which distributes information about the number of subscriptions and open files on or off."
871 "|Enable[bool]:Enable of disable NUM_SUBS service (yes/no).");
872
873 AddEvent("SET_RUN_TIMEOUT", "L:1", kSM_Ready, kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun)
874 (bind(&DataLogger::SetRunTimeoutDelay, this, placeholders::_1))
875 ("Set the timeout delay for old run numbers."
876 "|timeout[min]:Time out in minutes after which files for expired runs are closed.");
877
878 fDestructing = false;
879
880 fPreviousOldRunNumberCheck = Time().Mjd();
881
882 fDailyFileDayChangedAlready = true;
883 fRunNumberTimeout = 60000; //default run-timeout set to 1 minute
884 fRunNumber.push_back(RunNumberType());
885 fRunNumber.back().runNumber = -1;
886 fRunNumber.back().time = Time();
887 NotifyOpenedFile("", 0, fOpenedNightlyFiles);
888 NotifyOpenedFile("", 0, fOpenedRunFiles);
889
890 fRunNumberService = NULL;
891 fShouldAutoStart = false;
892 fAutoStarted = false;
893 if(fDebugIsOn)
894 {
895 Debug("DataLogger Init Done.");
896 }
897}
898
899// --------------------------------------------------------------------------
900//
901//! Destructor
902//
903DataLogger::~DataLogger()
904{
905 if (fDebugIsOn)
906 Debug("DataLogger destruction starts");
907
908 //this boolean should not be required anymore
909 fDestructing = true;
910
911 //first, let's backup the datalogger/message service subscription
912// shared_ptr<DimStampedInfo> messageBackup;
913// const SubscriptionsListType::iterator x = fServiceSubscriptions.find("DATA_LOGGER");
914// if (x != fServiceSubscriptions.end())
915// {
916// const map<string, SubscriptionType>::iterator y = x->second.find("MESSAGE");
917// if (y != x->second.end())
918// messageBackup = y->second.dimInfo;
919// }
920
921
922 //now clear the services subscriptions
923 dim_lock();
924 fServiceSubscriptions.clear();
925 dim_unlock();
926
927 //clear any remaining run number (should remain only one)
928 while (fRunNumber.size() > 0)
929 {
930 RemoveOldestRunNumber();
931 }
932 //go to the ready state. i.e. close all files, run-wise first
933 GoToReady();
934
935
936 Info("Will soon close the daily log file");
937
938 delete fOpenedNightlyFiles;
939 delete fOpenedRunFiles;
940 delete fNumSubAndFits;
941
942 //release message service before closing nightly log file
943// if (messageBackup)
944 // messageBackup.reset();
945
946 if (fNightlyLogFile.is_open())//this file is the only one that has not been closed by GoToReady
947 {
948// dim_lock();
949 fNightlyLogFile << endl;
950 fNightlyLogFile.close();
951// dim_unlock();
952 }
953
954 if (fDebugIsOn)
955 Debug("DataLogger desctruction ends");
956}
957
958// --------------------------------------------------------------------------
959//
960//! checks if old run numbers should be trimmed and if so, do it
961//
962void DataLogger::TrimOldRunNumbers()
963{
964 const Time cTime = Time();
965
966 if (cTime - fPreviousOldRunNumberCheck < boost::posix_time::milliseconds(fRunNumberTimeout))
967 return;
968
969 while (fRunNumber.size() > 1 && (cTime - fRunNumber.back().time) > boost::posix_time::milliseconds(fRunNumberTimeout))
970 {
971 RemoveOldestRunNumber();
972 }
973 fPreviousOldRunNumberCheck = cTime;
974}
975// --------------------------------------------------------------------------
976//
977//! Inherited from DimInfo. Handles all the Infos to which we subscribed, and log them
978//
979void DataLogger::infoHandler()
980{
981 if ((GetCurrentState() == kSM_Ready) && (!fAutoStarted) && fShouldAutoStart)
982 {
983 fAutoStarted = true;
984 SetCurrentState(Start());
985 }
986 else {if (GetCurrentState() > kSM_Ready)
987 fAutoStarted = true;
988 }
989 DimInfo* I = getInfo();
990
991 if (I==NULL)
992 return;
993
994 //check if the service pointer corresponds to something that we subscribed to
995 //this is a fix for a bug that provides bad Infos when a server starts
996 bool found = false;
997 SubscriptionsListType::iterator x;
998 map<string, SubscriptionType>::iterator y;
999 for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
1000 {//find current service is subscriptions
1001 for (y=x->second.begin(); y!=x->second.end();y++)
1002 if ((y->second.dimInfo).get() == I)
1003 {
1004 found = true;
1005 break;
1006 }
1007 if (found)
1008 break;
1009 }
1010 if (!found)
1011 return;
1012
1013 if (I->getSize() <= 0 || I->getData()==NULL)
1014 {
1015 return;
1016 }
1017 if (strlen(I->getFormat()) == 0)
1018 {
1019 ostringstream str;
1020 str << "Format of " << I->getName() << " is empty (ptr=" << I->getData() << ", size=" << I->getSize() << ")... ignoring it.";
1021 Warn(str);
1022 return;
1023 }
1024 // Make sure that getTimestampMillisecs is NEVER called before
1025 // getTimestamp is properly called
1026 // check that the message has been updated by something, i.e. must be different from its initial value
1027 if (I->getTimestamp() == 0)
1028 {
1029 return;
1030 }
1031// cout.precision(20);
1032// cout << "Orig timestamp: " << Time(I->getTimestamp(), I->getTimestampMillisecs()*1000).Mjd() << endl;
1033 // FIXME: Here we have to check if we have received the
1034 // service with the run-number.
1035 // CheckForRunNumber(I); has been removed because we have to
1036 // subscribe to this service anyway and hence we have the pointer
1037 // (no need to check for the name)
1038 CheckForRunNumber(I);
1039
1040 Report(I, y->second);
1041
1042 //remove old run numbers
1043 TrimOldRunNumbers();
1044}
1045
1046// --------------------------------------------------------------------------
1047//
1048//! Open the text files associated with the given run number
1049//! @param run the run number to be dealt with
1050//
1051int DataLogger::OpenRunFile(RunNumberType& run)
1052{
1053#ifdef RUN_LOGS
1054 // open log file
1055 run.logName = CompileFileName(fFilePath, "", "log", run.runNumber);
1056 if (!OpenStream(run.logFile, run.logName))
1057 return -1;
1058#endif
1059
1060 // open report file
1061 run.reportName = CompileFileNameWithPath(fFilePath, "", "rep", run.runNumber);
1062 if (!OpenStream(run.reportFile, run.reportName))
1063 return -1;
1064
1065 //get the size of the newly opened file.
1066#ifdef RUN_LOGS
1067 fFilesStats.FileOpened(run.logName);
1068#endif
1069 fFilesStats.FileOpened(run.reportName);
1070 //TODO this notification scheme might be messed up now.... fix it !
1071 const string baseFileName = CompileFileNameWithPath(fFilePath, "", "", run.runNumber);
1072 NotifyOpenedFile(baseFileName, 3, fOpenedRunFiles);
1073 run.openedFits.clear();
1074 return 0;
1075}
1076// --------------------------------------------------------------------------
1077//
1078//! Add a new active run number
1079//! @param newRun the new run number
1080//! @param time the time at which the new run number was issued
1081//
1082void DataLogger::AddNewRunNumber(int64_t newRun, Time time)
1083{
1084
1085 if (newRun > 0xffffffff)
1086 {
1087 Error("New run number too large, out of range. Ignoring.");
1088 return;
1089 }
1090 for (std::vector<int64_t>::const_iterator it=previousRunNumbers.begin(); it != previousRunNumbers.end(); it++)
1091 {
1092 if (*it == newRun)
1093 {
1094 Error("Newly provided run number has already been used (or is still in use). Going to error state");
1095 SetCurrentState(kSM_BadFolder);
1096 return;
1097 }
1098 }
1099 if (fDebugIsOn)
1100 {
1101 ostringstream str;
1102 str << "Adding new run number " << newRun << " issued at " << time;
1103 Debug(str);
1104 }
1105 //Add new run number to run number list
1106 fRunNumber.push_back(RunNumberType());
1107 fRunNumber.back().runNumber = int32_t(newRun);
1108 fRunNumber.back().time = time;
1109
1110 ostringstream str;
1111 str << "The new run number is: " << fRunNumber.back().runNumber;
1112 Message(str);
1113
1114 if (GetCurrentState() != kSM_Logging && GetCurrentState() != kSM_WaitingRun )
1115 return;
1116 //open the log and report files
1117 if (fRunNumber.back().runNumber > 0)
1118 if (OpenRunFile(fRunNumber.back()) != 0)
1119 {//an error occured. close current run files and go to error state
1120 for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
1121 {
1122 if (it->reportFile->is_open())
1123 {
1124 it->reportFile->close();
1125 Info("Closed: "+it->reportName);
1126 }
1127#ifdef RUN_LOGS
1128 if (it->logFile->is_open())
1129 {
1130 it->logFile->close();
1131 Info("Closed: "+it->logName);
1132 }
1133#endif
1134 }
1135 StopRunLogging();
1136 SetCurrentState(kSM_BadFolder);
1137 }
1138
1139 if (newRun > 0 && GetCurrentState() == kSM_WaitingRun)
1140 SetCurrentState(kSM_Logging);
1141 if (newRun < 0 && GetCurrentState() == kSM_Logging)
1142 SetCurrentState(kSM_WaitingRun);
1143}
1144// --------------------------------------------------------------------------
1145//
1146//! Checks whether or not the current info is a run number.
1147//! If so, then remember it. A run number is required to open the run-log file
1148//! @param I
1149//! the current DimInfo
1150//
1151void DataLogger::CheckForRunNumber(DimInfo* I)
1152{
1153 if (I != fRunNumberService)
1154 return;
1155
1156 AddNewRunNumber(I->getLonglong(), Time(I->getTimestamp(), I->getTimestampMillisecs()*1000));
1157}
1158
1159// --------------------------------------------------------------------------
1160//
1161//! write infos to log files.
1162//! @param I
1163//! The current DimInfo
1164//! @param sub
1165//! The dataLogger's subscription corresponding to this DimInfo
1166//
1167void DataLogger::Report(DimInfo* I, SubscriptionType& sub)
1168{
1169 const string fmt(I->getFormat());
1170
1171 const bool isItaReport = fmt!="C";
1172
1173 if (!fNightlyLogFile.is_open())
1174 return;
1175
1176 if (fDebugIsOn && string(I->getName())!="DATA_LOGGER/MESSAGE")
1177 {
1178 ostringstream str;
1179 str << "Logging " << I->getName() << " [" << I->getFormat() << "] (" << I->getSize() << ")";
1180 Debug(str);
1181 }
1182
1183 //
1184 // Check whether we should close and reopen daily text files or not
1185 // calculate time "centered" around noon instead of midnight
1186 // if number of days has changed, then files should be closed and reopenned.
1187 const Time timeNow;
1188 const Time nowMinusTwelve = timeNow-boost::posix_time::hours(12);
1189 int newDayNumber = nowMinusTwelve.M()*31 + nowMinusTwelve.D();//assume 31 days per month. we do not really care, only want unique number per day of the year
1190
1191 //also check if we should flush the nightly files
1192 if (lastFlush < timeNow-boost::posix_time::minutes(1))
1193 {
1194 lastFlush = timeNow;
1195 SubscriptionsListType::iterator x;
1196 map<string, SubscriptionType>::iterator y;
1197 for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
1198 {//find current service is subscriptions
1199 for (y=x->second.begin(); y!=x->second.end();y++)
1200 if (y->second.nightlyFile.IsOpen())
1201 {
1202 y->second.nightlyFile.Flush();
1203 }
1204 }
1205 if (fDebugIsOn)
1206 Debug("Just flushed nightly fits files to the disk");
1207 }
1208
1209// if (Time().h() == 12 && !fDailyFileDayChangedAlready)
1210 if (newDayNumber != fCurrentDay)
1211 {
1212 fCurrentDay = newDayNumber;
1213 //crawl through the subcriptions and close any open nightly file
1214 SubscriptionsListType::iterator x;
1215 map<string, SubscriptionType>::iterator y;
1216 for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
1217 {//find current service is subscriptions
1218 for (y=x->second.begin(); y!=x->second.end();y++)
1219 if (y->second.nightlyFile.IsOpen())
1220 {
1221 y->second.nightlyFile.Close();
1222 }
1223 }
1224
1225 if (fDebugIsOn)
1226 Debug("Day have changed! Closing and reopening nightly files");
1227
1228 fNightlyLogFile << endl;
1229 fNightlyLogFile.close();
1230 fNightlyReportFile.close();
1231
1232 Info("Closed: "+fFullNightlyLogFileName);
1233 Info("Closed: "+fFullNightlyReportFileName);
1234
1235 fFullNightlyLogFileName = CompileFileNameWithPath(fFilePath, "", "log");
1236 if (!OpenTextFile(fNightlyLogFile, fFullNightlyLogFileName))
1237 {
1238 GoToReady();
1239 SetCurrentState(kSM_BadFolder);
1240 return;
1241 }
1242 fNightlyLogFile << endl;
1243
1244 fFullNightlyReportFileName = CompileFileNameWithPath(fFilePath, "", "rep");
1245 if (!OpenTextFile(fNightlyReportFile, fFullNightlyReportFileName))
1246 {
1247 GoToReady();
1248 SetCurrentState(kSM_BadFolder);
1249 return;
1250 }
1251
1252 // fDailyFileDayChangedAlready = true;
1253 }
1254// if (Time().h() != 12 && fDailyFileDayChangedAlready)
1255// fDailyFileDayChangedAlready = false;
1256
1257 //create the converter for that service
1258 if (!sub.fConv)
1259 {
1260 sub.fConv = shared_ptr<Converter>(new Converter(Out(), I->getFormat()));
1261 if (!sub.fConv->valid())
1262 {
1263 ostringstream str;
1264 str << "Couldn't properly parse the format... service " << sub.dimInfo->getName() << " ignored.";
1265 Error(str);
1266 return;
1267 }
1268 }
1269 //construct the header
1270 ostringstream header;
1271 const Time cTime(I->getTimestamp(), I->getTimestampMillisecs()*1000);
1272 fQuality = I->getQuality();
1273 fMjD = cTime.Mjd() ? cTime.Mjd()-40587 : 0;
1274
1275 //figure out which run file should be used
1276 ofstream* targetRunFile = NULL;
1277 RunNumberType* cRunNumber = NULL;
1278 if (GetCurrentState() == kSM_Logging || GetCurrentState() == kSM_WaitingRun)
1279 {
1280 list<RunNumberType>::reverse_iterator rit;
1281 for (rit=fRunNumber.rbegin(); rit!=fRunNumber.rend(); rit++)
1282 {
1283 if (rit->time < cTime) //this is the run number that we want to use
1284 {
1285 //Find something better to convert iterator to pointer than the ugly line below....
1286 cRunNumber = &(*rit);
1287 sub.runNumber = rit->runNumber;
1288
1289 if (rit->runNumber <= 0)//take only positive numbers.
1290 break;
1291#ifdef RUN_LOGS
1292 targetRunFile = isItaReport ? (rit->reportFile).get() : (rit->logFile).get();
1293#else
1294 targetRunFile = isItaReport ? (rit->reportFile).get() : NULL;
1295#endif
1296 break;
1297 }
1298 }
1299 if (rit == fRunNumber.rend() && fRunNumber.size() != 0)
1300 {
1301 Error("Could not find an appropriate run number for info coming at time "+cTime.GetAsStr());
1302 Error("Active run numbers: ");
1303 for (rit=fRunNumber.rbegin(); rit != fRunNumber.rend(); rit++)
1304 {
1305 ostringstream str;
1306 str << " -> " << rit->runNumber;
1307 Error(str);
1308 }
1309
1310 }
1311 }
1312
1313 if (isItaReport)
1314 {
1315 //write text header
1316 header << I->getName() << " " << fQuality << " ";
1317 header << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
1318 header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
1319 header << cTime.ms() << " " << I->getTimestamp() << " ";
1320
1321 string text;
1322 try
1323 {
1324 text = sub.fConv->GetString(I->getData(), I->getSize());
1325 }
1326 catch (const runtime_error &e)
1327 {
1328 ostringstream str;
1329 str << "Parsing service " << sub.dimInfo->getName();
1330 str << " failed: " << e.what() << " removing the subscription for now.";
1331 Error(str);
1332 //remove this subscription from the list.
1333 //because these operators use references to elements, and because they're supposed here to erase these objects on the way, I'm not too sure... so duplicate the names !
1334 RemoveService(sub.server, sub.service, false);
1335// string server = sub.server;
1336// string service = sub.service;
1337// fServiceSubscriptions.find(server)->second.erase(service);
1338 return;
1339 }
1340
1341 if (text.empty())
1342 {
1343 ostringstream str;
1344 str << "Service " << sub.dimInfo->getName() << " sent an empty string";
1345 Info(str);
1346 return;
1347 }
1348 //replace bizarre characters by white space
1349 replace(text.begin(), text.end(), '\n', '\\');
1350 replace_if(text.begin(), text.end(), ptr_fun<int, int>(&iscntrl), ' ');
1351
1352 //write entry to Nightly report
1353 if (fNightlyReportFile.is_open())
1354 {
1355 fNightlyReportFile << header.str() << text << endl;
1356 if (!CheckForOfstreamError(fNightlyReportFile, true))
1357 return;
1358 }
1359 //write entry to run-report
1360 if (targetRunFile && targetRunFile->is_open())
1361 {
1362 *targetRunFile << header.str() << text << endl;
1363 if (!CheckForOfstreamError(*targetRunFile, false))
1364 return;
1365 }
1366#ifdef HAVE_FITS
1367 //check if the last received event was before noon and if current one is after noon.
1368 //if so, close the file so that it gets reopened.
1369// if (sub.nightlyFile.IsOpen())
1370// if ((sub.lastReceivedEvent != Time::None) && (sub.lastReceivedEvent.h() < 12) && (cTime.h() >= 12))
1371// {
1372// sub.nightlyFile.Close();
1373// }
1374 sub.lastReceivedEvent = cTime;//ici
1375 if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen() || sub.runNumber != sub.runFile.fRunNumber)
1376 if (GetCurrentState() != kSM_Ready)
1377 OpenFITSFiles(sub, cRunNumber);
1378 WriteToFITS(sub);
1379#endif
1380 }
1381 else
1382 {//write entry to both Nightly and run logs
1383 vector<string> strings;
1384 try
1385 {
1386 strings = sub.fConv->ToStrings(I->getData());
1387 }
1388 catch (const runtime_error &e)
1389 {
1390 ostringstream str;
1391 str << "Parsing service " << sub.dimInfo->getName();
1392 str << " failed: " << e.what() << " removing the subscription for now.";
1393 Error(str);
1394 //remove this subscription from the list.
1395 //because these operators use references to elements, and because they're supposed here to erase these objects on the way, I'm not too sure... so duplicate the names !
1396 RemoveService(sub.server, sub.service, false);
1397// string server = sub.server;
1398// string service = sub.service;
1399// fServiceSubscriptions.find(server)->second.erase(service);
1400 return;
1401 }
1402 if (strings.size() > 1)
1403 {
1404 ostringstream err;
1405 err << "There was more than one string message in service " << I->getName() << " going to fatal error state";
1406 Error(err.str());
1407 }
1408 ostringstream msg;
1409 msg << I->getName() << ": " << strings[0];
1410
1411 if (fNightlyLogFile.is_open())
1412 {
1413 MessageImp(fNightlyLogFile).Write(cTime, msg.str().c_str(), kMessage);//fQuality); Silently ignore QoS in case of messages
1414 if (!CheckForOfstreamError(fNightlyLogFile, true))
1415 return;
1416 }
1417 if (targetRunFile && targetRunFile->is_open())
1418 {
1419 MessageImp(*targetRunFile).Write(cTime, msg.str().c_str(), kMessage);//fQuality);
1420 if (!CheckForOfstreamError(*targetRunFile, false))
1421 return;
1422 }
1423
1424 sub.lastReceivedEvent = cTime;//ici
1425 if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen() || sub.runNumber != sub.runFile.fRunNumber)
1426 if (GetCurrentState() != kSM_Ready)
1427 OpenFITSFiles(sub, cRunNumber);
1428 WriteToFITS(sub);
1429 }
1430
1431}
1432
1433// --------------------------------------------------------------------------
1434//
1435//! print the dataLogger's current state. invoked by the PRINT command
1436//! @param evt
1437//! the current event. Not used by the method
1438//! @returns
1439//! the new state. Which, in that case, is the current state
1440//!
1441int DataLogger::PrintState(const Event& )
1442{
1443 Message("------------------------------------------");
1444 Message("------- DATA LOGGER CURRENT STATE --------");
1445 Message("------------------------------------------");
1446
1447 //print the path configuration
1448#if BOOST_VERSION < 104600
1449 Message("File path: " + boost::filesystem::system_complete(boost::filesystem::path(fFilePath)).directory_string());
1450#else
1451 Message("File path: " + boost::filesystem::system_complete(boost::filesystem::path(fFilePath)).parent_path().string());
1452#endif
1453
1454 //print active run numbers
1455 ostringstream str;
1456 //timeout value
1457 str << "Timeout delay for old run numbers: " << fRunNumberTimeout << " ms";
1458 Message(str);
1459 str.str("");
1460 str << "Active Run Numbers:";
1461 for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
1462 str << " " << it->runNumber;
1463 if (fRunNumber.size()==0)
1464 str << " <none>";
1465 Message(str);
1466
1467 //print all the open files.
1468 Message("------------ OPEN FILES ----------------");
1469 if (fNightlyLogFile.is_open())
1470 Message("Nightly log-file: "+fFullNightlyLogFileName);
1471
1472 if (fNightlyReportFile.is_open())
1473 Message("Nightly report-file: "+fFullNightlyReportFileName);
1474
1475 for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it!=fRunNumber.end(); it++)
1476 {
1477#ifdef RUN_LOGS
1478 if (it->logFile->is_open())
1479 Message("Run log-file: " + it->logName);
1480#endif
1481 if (it->reportFile->is_open())
1482 Message("Run report-file: " + it->reportName);
1483 }
1484
1485 const DimWriteStatistics::Stats statVar = fFilesStats.GetTotalSizeWritten();
1486 // /*const bool statWarning =*/ calculateTotalSizeWritten(statVar, true);
1487#ifdef HAVE_FITS
1488 str.str("");
1489 str << "Number of open FITS files: " << fNumSubAndFitsData.numOpenFits;
1490 Message(str);
1491 // FIXME: Print list of open FITS files
1492#else
1493 Message("FITS output disabled at compilation");
1494#endif
1495 Message("----------------- STATS ------------------");
1496 if (fFilesStats.GetUpdateInterval()>0)
1497 {
1498 str.str("");
1499 str << "Statistics are updated every " << fFilesStats.GetUpdateInterval() << " ms";
1500 Message(str);
1501 }
1502 else
1503 Message("Statistics updates are currently disabled.");
1504 str.str("");
1505 str << "Total Size written: " << statVar.sizeWritten/1000 << " kB";
1506 Message(str);
1507 str.str("");
1508 str << "Disk free space: " << statVar.freeSpace/1000000 << " MB";
1509 Message(str);
1510
1511 Message("------------ DIM SUBSCRIPTIONS -----------");
1512 str.str("");
1513 str << "There are " << fNumSubAndFitsData.numSubscriptions << " active DIM subscriptions.";
1514 Message(str);
1515 for (map<const string, map<string, SubscriptionType> >::const_iterator it=fServiceSubscriptions.begin(); it!= fServiceSubscriptions.end();it++)
1516 {
1517 Message("Server "+it->first);
1518 for (map<string, SubscriptionType>::const_iterator it2=it->second.begin(); it2!=it->second.end(); it2++)
1519 Message(" -> "+it2->first);
1520 }
1521 Message("--------------- BLOCK LIST ---------------");
1522 for (set<string>::const_iterator it=fBlackList.begin(); it != fBlackList.end(); it++)
1523 Message(" -> "+*it);
1524 if (fBlackList.size()==0)
1525 Message(" <empty>");
1526
1527 Message("--------------- ALLOW LIST ---------------");
1528 for (set<string>::const_iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++)
1529 Message(" -> "+*it);
1530 if (fWhiteList.size()==0)
1531 Message(" <empty>");
1532
1533 Message("-------------- GROUPING LIST -------------");
1534 Message("The following servers and/or services will");
1535 Message("be grouped into a single fits file:");
1536 for (set<string>::const_iterator it=fGrouping.begin(); it != fGrouping.end(); it++)
1537 Message(" -> "+*it);
1538 if (fGrouping.size()==0)
1539 Message(" <no grouping>");
1540
1541 Message("------------------------------------------");
1542 Message("-------- END OF DATA LOGGER STATE --------");
1543 Message("------------------------------------------");
1544
1545 return GetCurrentState();
1546}
1547
1548// --------------------------------------------------------------------------
1549//
1550//! turn debug mode on and off
1551//! @param evt
1552//! the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1
1553//! @returns
1554//! the new state. Which, in that case, is the current state
1555//!
1556int DataLogger::SetDebugOnOff(const Event& evt)
1557{
1558 const bool backupDebug = fDebugIsOn;
1559
1560 fDebugIsOn = evt.GetBool();
1561
1562 if (fDebugIsOn == backupDebug)
1563 Message("Debug mode was already in the requested state.");
1564
1565 ostringstream str;
1566 str << "Debug mode is now " << fDebugIsOn;
1567 Message(str);
1568
1569 fFilesStats.SetDebugMode(fDebugIsOn);
1570
1571 return GetCurrentState();
1572}
1573// --------------------------------------------------------------------------
1574//
1575//! set the statistics update period duration. 0 disables the statistics
1576//! @param evt
1577//! the current event. contains the new duration.
1578//! @returns
1579//! the new state. Which, in that case, is the current state
1580//!
1581int DataLogger::SetStatsPeriod(const Event& evt)
1582{
1583 fFilesStats.SetUpdateInterval(evt.GetShort());
1584 return GetCurrentState();
1585}
1586// --------------------------------------------------------------------------
1587//
1588//! set the opened files service on or off.
1589//! @param evt
1590//! the current event. contains the instruction string. similar to setdebugonoff
1591//! @returns
1592//! the new state. Which, in that case, is the current state
1593//!
1594int DataLogger::SetOpenedFilesOnOff(const Event& evt)
1595{
1596 const bool backupOpened = fOpenedFilesIsOn;
1597
1598 fOpenedFilesIsOn = evt.GetBool();
1599
1600 if (fOpenedFilesIsOn == backupOpened)
1601 Message("Opened files service mode was already in the requested state.");
1602
1603 ostringstream str;
1604 str << "Opened files service mode is now " << fOpenedFilesIsOn;
1605 Message(str);
1606
1607 return GetCurrentState();
1608}
1609
1610// --------------------------------------------------------------------------
1611//
1612//! set the number of subscriptions and opened fits on and off
1613//! @param evt
1614//! the current event. contains the instruction string. similar to setdebugonoff
1615//! @returns
1616//! the new state. Which, in that case, is the current state
1617//!
1618int DataLogger::SetNumSubsAndFitsOnOff(const Event& evt)
1619{
1620 const bool backupSubs = fNumSubAndFitsIsOn;
1621
1622 fNumSubAndFitsIsOn = evt.GetBool();
1623
1624 if (fNumSubAndFitsIsOn == backupSubs)
1625 Message("Number of subscriptions service mode was already in the requested state");
1626
1627 ostringstream str;
1628 str << "Number of subscriptions service mode is now " << fNumSubAndFitsIsOn;
1629 Message(str);
1630
1631 return GetCurrentState();
1632}
1633// --------------------------------------------------------------------------
1634//
1635//! set the timeout delay for old run numbers
1636//! @param evt
1637//! the current event. contains the timeout delay long value
1638//! @returns
1639//! the new state. Which, in that case, is the current state
1640//!
1641int DataLogger::SetRunTimeoutDelay(const Event& evt)
1642{
1643 if (evt.GetUInt() == 0)
1644 {
1645 Error("Timeout delays for old run numbers must be greater than 0... ignored.");
1646 return GetCurrentState();
1647 }
1648
1649 if (fRunNumberTimeout == evt.GetUInt())
1650 Message("New timeout for old run numbers is same value as previous one.");
1651
1652 fRunNumberTimeout = evt.GetUInt();
1653
1654 ostringstream str;
1655 str << "Timeout delay for old run numbers is now " << fRunNumberTimeout << " ms";
1656 Message(str);
1657
1658 return GetCurrentState();
1659}
1660
1661// --------------------------------------------------------------------------
1662//
1663//! Sets the path to use for the Nightly log file.
1664//! @param evt
1665//! the event transporting the path
1666//! @returns
1667//! currently only the current state.
1668/*
1669int DataLogger::ConfigureFilePath(const Event& evt)
1670{
1671 if (!evt.GetText())
1672 {
1673 Error("Empty folder given. Please specify a valid path.");
1674 return GetCurrentState();
1675 }
1676
1677 const string givenPath = evt.GetText();
1678 if (!DoesPathExist(givenPath))
1679 {
1680 Error("Provided path '"+givenPath+"' is not a valid folder... ignored.");
1681 return GetCurrentState();
1682 }
1683
1684 Message("New folder: "+givenPath);
1685
1686 fFilePath = givenPath;
1687
1688 fFilesStats.SetCurrentFolder(givenPath);
1689
1690 return GetCurrentState();
1691}
1692*/
1693
1694// --------------------------------------------------------------------------
1695//
1696//! Notifies the DIM service that a particular file was opened
1697//! @ param name the base name of the opened file, i.e. without path nor extension.
1698//! WARNING: use string instead of string& because I pass values that do not convert to string&.
1699//! this is not a problem though because file are not opened so often.
1700//! @ param type the type of the opened file. 0 = none open, 1 = log, 2 = text, 4 = fits
1701inline void DataLogger::NotifyOpenedFile(const string &name, int type, DimDescribedService* service)
1702{
1703 if (!fOpenedFilesIsOn)
1704 return;
1705
1706 if (fDebugIsOn)
1707 {
1708 ostringstream str;
1709 str << "Updating " << service->getName() << " file '" << name << "' (type=" << type << ")";
1710 Debug(str);
1711
1712 str.str("");
1713 str << "Num subscriptions: " << fNumSubAndFitsData.numSubscriptions << " Num open FITS files: " << fNumSubAndFitsData.numOpenFits;
1714 Debug(str);
1715 }
1716
1717 if (name.size()+1 > FILENAME_MAX)
1718 {
1719 Error("Provided file name '" + name + "' is longer than allowed file name length.");
1720 return;
1721 }
1722
1723 OpenFileToDim fToDim;
1724 fToDim.code = type;
1725 memcpy(fToDim.fileName, name.c_str(), name.size()+1);
1726
1727 service->setData(reinterpret_cast<void*>(&fToDim), name.size()+1+sizeof(uint32_t));
1728 service->setQuality(0);
1729 service->Update();
1730}
1731// --------------------------------------------------------------------------
1732//
1733//! Implements the Start transition.
1734//! Concatenates the given path for the Nightly file and the filename itself (based on the day),
1735//! and tries to open it.
1736//! @returns
1737//! kSM_NightlyOpen if success, kSM_BadFolder if failure
1738int DataLogger::Start()
1739{
1740 if (fDebugIsOn)
1741 {
1742 Debug("Starting...");
1743 }
1744 fFullNightlyLogFileName = CompileFileNameWithPath(fFilePath, "", "log");
1745 bool nightlyLogOpen = fNightlyLogFile.is_open();
1746 if (!OpenTextFile(fNightlyLogFile, fFullNightlyLogFileName))
1747 return kSM_BadFolder;
1748 if (!nightlyLogOpen)
1749 fNightlyLogFile << endl;
1750
1751 fFullNightlyReportFileName = CompileFileNameWithPath(fFilePath, "", "rep");
1752 if (!OpenTextFile(fNightlyReportFile, fFullNightlyReportFileName))
1753 {
1754 fNightlyLogFile.close();
1755 Info("Closed: "+fFullNightlyReportFileName);
1756 return kSM_BadFolder;
1757 }
1758
1759 fFilesStats.FileOpened(fFullNightlyLogFileName);
1760 fFilesStats.FileOpened(fFullNightlyReportFileName);
1761 //notify that a new file has been opened.
1762 const string baseFileName = CompileFileNameWithPath(fFilePath, "", "");
1763 NotifyOpenedFile(baseFileName, 3, fOpenedNightlyFiles);
1764
1765 fOpenedNightlyFits.clear();
1766
1767 return kSM_NightlyOpen;
1768}
1769
1770#ifdef HAVE_FITS
1771// --------------------------------------------------------------------------
1772//
1773//! open if required a the FITS files corresponding to a given subscription
1774//! @param sub
1775//! the current DimInfo subscription being examined
1776void DataLogger::OpenFITSFiles(SubscriptionType& sub, RunNumberType* cRunNumber)
1777{
1778 string serviceName(sub.dimInfo->getName());
1779
1780 //if run number has changed, reopen a new fits file with the correct run number.
1781 if (sub.runFile.IsOpen() && sub.runFile.fRunNumber != sub.runNumber)
1782 {
1783 sub.runFile.Close();
1784 Info("Closed: "+sub.runFile.GetName()+" (new run number)");
1785 }
1786
1787 //we must check if we should group this service subscription to a single fits file before we replace the / by _
1788 bool hasGrouping = false;
1789 if (!sub.runFile.IsOpen() && ((GetCurrentState() == kSM_Logging) || (GetCurrentState() == kSM_WaitingRun)))
1790 {//will we find this service in the grouping list ?
1791 for (set<string>::const_iterator it=fGrouping.begin(); it!=fGrouping.end(); it++)
1792 {
1793 if (serviceName.find(*it) != string::npos)
1794 {
1795 hasGrouping = true;
1796 break;
1797 }
1798 }
1799 }
1800 for (unsigned int i=0;i<serviceName.size(); i++)
1801 {
1802 if (serviceName[i] == '/')
1803 {
1804 serviceName[i] = '_';
1805 break;
1806 }
1807 }
1808 //we open the NightlyFile anyway, otherwise this function shouldn't have been called.
1809 if (!sub.nightlyFile.IsOpen())
1810 {
1811 const string partialName = CompileFileNameWithPath(fFilePath, serviceName, "fits");
1812
1813 const string fileNameOnly = partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1814 if (!sub.fitsBufferAllocated)
1815 AllocateFITSBuffers(sub);
1816 //get the size of the file we're about to open
1817 if (fFilesStats.FileOpened(partialName))
1818 fOpenedNightlyFits[fileNameOnly].push_back(serviceName);
1819
1820 if (!sub.nightlyFile.Open(partialName, serviceName, &fNumSubAndFitsData.numOpenFits, this, 0))
1821 {
1822 GoToRunWriteErrorState();
1823 return;
1824 }
1825
1826 ostringstream str;
1827 str << "Opened: " << partialName << " (Nfits=" << fNumSubAndFitsData.numOpenFits << ")";
1828 Info(str);
1829
1830 //notify the opening
1831 const string baseFileName = CompileFileNameWithPath(fFilePath, "", "");
1832 NotifyOpenedFile(baseFileName, 7, fOpenedNightlyFiles);
1833 if (fNumSubAndFitsIsOn)
1834 fNumSubAndFits->Update();
1835 }
1836 //do the actual file open
1837 if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_WaitingRun || GetCurrentState() == kSM_Logging) && sub.runNumber > 0)
1838 {//buffer for the run file have already been allocated when doing the Nightly file
1839
1840 const string partialName =
1841 CompileFileNameWithPath(fFilePath, hasGrouping ? "" : serviceName, "fits", sub.runNumber);
1842
1843 const string fileNameOnly =
1844 partialName.substr(partialName.find_last_of('/')+1, partialName.size());
1845
1846 //get the size of the file we're about to open
1847 if (fFilesStats.FileOpened(partialName))
1848 cRunNumber->openedFits[fileNameOnly].push_back(serviceName);
1849 else
1850 if (hasGrouping)
1851 {
1852 cRunNumber->addServiceToOpenedFits(fileNameOnly, serviceName);
1853 }
1854
1855 if (hasGrouping && (!cRunNumber->runFitsFile.get()))
1856 try
1857 {
1858 cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>(new CCfits::FITS(partialName, CCfits::RWmode::Write));
1859 (fNumSubAndFitsData.numOpenFits)++;
1860 }
1861 catch (CCfits::FitsException e)
1862 {
1863 ostringstream str;
1864 str << "Open FITS file " << partialName << ": " << e.message();
1865 Error(str);
1866 cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>();
1867 GoToRunWriteErrorState();
1868 return;
1869 }
1870
1871 const string baseFileName = CompileFileNameWithPath(fFilePath, "", "", sub.runNumber);
1872 NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles);
1873
1874 if (hasGrouping)
1875 {
1876 if (!sub.runFile.Open(partialName, serviceName, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber, cRunNumber->runFitsFile.get()))
1877 {
1878 GoToRunWriteErrorState();
1879 return;
1880 }
1881 }
1882 else
1883 {
1884 if (!sub.runFile.Open(partialName, serviceName, &fNumSubAndFitsData.numOpenFits, this, sub.runNumber))
1885 {
1886 GoToRunWriteErrorState();
1887 return;
1888 }
1889 }
1890
1891 ostringstream str;
1892 str << "Opened: " << partialName << " (Nfits=" << fNumSubAndFitsData.numOpenFits << ")";
1893 Info(str);
1894
1895 if (fNumSubAndFitsIsOn)
1896 fNumSubAndFits->Update();
1897 }
1898}
1899// --------------------------------------------------------------------------
1900//
1901//! Allocates the required memory for a given pair of fits files (nightly and run)
1902//! @param sub the subscription of interest.
1903//
1904void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
1905{
1906 //Init the time columns of the file
1907 Description dateDesc(string("Time"), string("Modified Julian Date"), string("MJD"));
1908 sub.nightlyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1909 sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
1910
1911 Description QoSDesc("QoS", "Quality of service", "");
1912 sub.nightlyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1913 sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
1914
1915 // Compilation failed
1916 if (!sub.fConv->valid())
1917 {
1918 Error("Compilation of format string failed.");
1919 return;
1920 }
1921
1922 //we've got a nice structure describing the format of this service's messages.
1923 //Let's create the appropriate FITS columns
1924 const vector<string> dataFormatsLocal = sub.fConv->GetFitsFormat();
1925
1926 ostringstream str;
1927 str << "Initializing data columns for service " << sub.server << "/" << sub.service;
1928 Info(str);
1929 sub.nightlyFile.InitDataColumns(GetDescription(sub.server, sub.service), dataFormatsLocal, sub.dimInfo->getData(), this);
1930 sub.runFile.InitDataColumns(GetDescription(sub.server, sub.service), dataFormatsLocal, sub.dimInfo->getData(), this);
1931
1932 sub.fitsBufferAllocated = true;
1933}
1934// --------------------------------------------------------------------------
1935//
1936//! write a dimInfo data to its corresponding FITS files
1937//
1938void DataLogger::WriteToFITS(SubscriptionType& sub)
1939{
1940 //nightly File status (open or not) already checked
1941 if (sub.nightlyFile.IsOpen())
1942 {
1943 if (!sub.nightlyFile.Write(*sub.fConv.get()))
1944 {
1945 //sub.nightlyFile.Close();
1946 RemoveService(sub.server, sub.service, false);
1947 GoToNightlyWriteErrorState();
1948 return;
1949 }
1950 // sub.nightlyFile.Flush();
1951 }
1952
1953 if (sub.runFile.IsOpen())
1954 {
1955 if (!sub.runFile.Write(*sub.fConv.get()))
1956 {
1957 sub.runFile.Close();
1958 RemoveService(sub.server, sub.service, false);
1959 GoToRunWriteErrorState();
1960 return;
1961 }
1962 }
1963}
1964#endif //if has_fits
1965// --------------------------------------------------------------------------
1966//
1967//! Go to Run Write Error State
1968// A write error has occurred. Checks what is the current state and take appropriate action
1969void DataLogger::GoToRunWriteErrorState()
1970{
1971 if ((GetCurrentState() != kSM_RunWriteError) &&
1972 (GetCurrentState() != kSM_DailyWriteError))
1973 SetCurrentState(kSM_RunWriteError);
1974}
1975// --------------------------------------------------------------------------
1976//
1977//! Go to Nightly Write Error State
1978// A write error has occurred. Checks what is the current state and take appropriate action
1979void DataLogger::GoToNightlyWriteErrorState()
1980{
1981 if (GetCurrentState() != kSM_DailyWriteError)
1982 SetCurrentState(kSM_DailyWriteError);
1983}
1984
1985/*
1986// --------------------------------------------------------------------------
1987//
1988//! Implements the StartRun transition.
1989//! Concatenates the given path for the run file and the filename itself (based on the run number),
1990//! and tries to open it.
1991//! @returns
1992//! kSM_Logging if success, kSM_BadRunConfig if failure.
1993int DataLogger::StartRun()
1994{
1995 if (fDebugIsOn)
1996 {
1997 Debug("Starting Run Logging...");
1998 }
1999 //open all the relevant run-files. i.e. all the files associated with run numbers.
2000 for (list<RunNumberType>::iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
2001 if (OpenRunFile(*it) != 0)
2002 {
2003 StopRun();
2004 return kSM_BadRunConfig;
2005 }
2006
2007 return kSM_Logging;
2008}
2009*/
2010#ifdef HAVE_FITS
2011// --------------------------------------------------------------------------
2012//
2013//! Create a fits group file with all the run-fits that were written (either daily or run)
2014//! @param filesToGroup a map of filenames mapping to table names to be grouped (i.e. a
2015//! single file can contain several tables to group
2016//! @param runNumber the run number that should be used for grouping. 0 means nightly group
2017//
2018void DataLogger::CreateFitsGrouping(map<string, vector<string> > & filesToGroup, int runNumber)
2019{
2020 if (fDebugIsOn)
2021 {
2022 ostringstream str;
2023 str << "Creating fits group for ";
2024 if (runNumber != 0)
2025 str << "run files";
2026 else
2027 str << "nightly files";
2028 Debug(str);
2029 }
2030 //create the FITS group corresponding to the ending run.
2031 CCfits::FITS* groupFile;
2032 unsigned int numFilesToGroup = 0;
2033 unsigned int maxCharLength = 0;
2034 for (map<string, vector<string> >::const_iterator it=filesToGroup.begin(); it != filesToGroup.end(); it++)
2035 {
2036 //add the number of tables in this file to the total number to group
2037 numFilesToGroup += it->second.size();
2038 //check the length of all the strings to be written, to determine the max string length to write
2039 if (it->first.size() > maxCharLength)
2040 maxCharLength = it->first.size();
2041 for (vector<string>::const_iterator jt=it->second.begin(); jt != it->second.end(); jt++)
2042 if (jt->size() > maxCharLength)
2043 maxCharLength = jt->size();
2044 }
2045
2046 if (fDebugIsOn)
2047 {
2048 ostringstream str;
2049 str << "There are " << numFilesToGroup << " tables to group";
2050 Debug(str);
2051 }
2052 if (numFilesToGroup <= 1)
2053 {
2054 filesToGroup.clear();
2055 return;
2056 }
2057 const string groupName = CompileFileNameWithPath(fFilePath, "", "fits", runNumber);
2058
2059 Info("Creating FITS group in: "+groupName);
2060
2061 CCfits::Table* groupTable;
2062// const int maxCharLength = FILENAME_MAX;
2063 try
2064 {
2065 groupFile = new CCfits::FITS(groupName, CCfits::RWmode::Write);
2066 //setup the column names
2067 ostringstream pathTypeName;
2068 pathTypeName << maxCharLength << "A";
2069 vector<string> names;
2070 vector<string> dataTypes;
2071 names.push_back("MEMBER_XTENSION");
2072 dataTypes.push_back("8A");
2073 names.push_back("MEMBER_URI_TYPE");
2074 dataTypes.push_back("3A");
2075 names.push_back("MEMBER_LOCATION");
2076 dataTypes.push_back(pathTypeName.str());
2077 names.push_back("MEMBER_NAME");
2078 dataTypes.push_back(pathTypeName.str());
2079 names.push_back("MEMBER_VERSION");
2080 dataTypes.push_back("1J");
2081 names.push_back("MEMBER_POSITION");
2082 dataTypes.push_back("1J");
2083
2084 groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
2085//TODO handle the case when the logger was stopped and restarted during the same day, i.e. the grouping file must be updated
2086 }
2087 catch (CCfits::FitsException e)
2088 {
2089 ostringstream str;
2090 str << "Creating FITS table GROUPING in " << groupName << ": " << e.message();
2091 Error(str);
2092 return;
2093 }
2094 try
2095 {
2096 groupTable->addKey("GRPNAME", "FACT_RAW_DATA", "Data from the FACT telescope");
2097 }
2098 catch (CCfits::FitsException e)
2099 {
2100 Error("CCfits::Table::addKey failed for 'GRPNAME' in '"+groupName+"-GROUPING': "+e.message());
2101 return;
2102 }
2103 //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about.
2104 //use cfitsio routines instead
2105 groupTable->makeThisCurrent();
2106 //create appropriate buffer.
2107 const unsigned int n = 8 + 3 + 2*maxCharLength + 1 + 8; //+1 for trailling character
2108
2109 vector<char> realBuffer(n);
2110
2111 char *startOfExtension = realBuffer.data();
2112 char *startOfURI = realBuffer.data()+8;
2113 char *startOfLocation = realBuffer.data()+8+3;
2114 char *startOfName = realBuffer.data()+8+3+maxCharLength;
2115 // char* startOfMemVer = reinterpret_cast<char*>(&fitsBuffer[8+3+2*maxCharLength]);
2116 // char* startOfMemPos = reinterpret_cast<char*>(&fitsBuffer[8+3+2*maxCharLength+1]);
2117
2118 strcpy(startOfExtension, "BINTABLE");
2119 strcpy(startOfURI, "URL");
2120
2121 realBuffer[8+3+2*maxCharLength+3] = 1;
2122 realBuffer[8+3+2*maxCharLength+7] = 1;
2123
2124 int i=1;
2125 for (map<string, vector<string> >::const_iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++)
2126 for (vector<string>::const_iterator jt=it->second.begin(); jt != it->second.end(); jt++, i++)
2127 {
2128 memset(startOfLocation, 0, 2*maxCharLength+1+8);
2129
2130 strcpy(startOfLocation, it->first.c_str());
2131 strcpy(startOfName, jt->c_str());
2132
2133 if (fDebugIsOn)
2134 {
2135 ostringstream str;
2136 str << "Grouping " << it->first << " " << *jt;
2137 Debug(str);
2138 }
2139
2140 int status = 0;
2141 fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength +8,
2142 reinterpret_cast<unsigned char*>(realBuffer.data()), &status);
2143 if (status)
2144 {
2145 char text[30];//max length of cfitsio error strings (from doc)
2146 fits_get_errstatus(status, text);
2147 ostringstream str;
2148 str << "Writing FITS row " << i << " in " << groupName << ": " << text << " (file_write_tblbytes, rc=" << status << ")";
2149 Error(str);
2150 GoToRunWriteErrorState();
2151 delete groupFile;
2152 return;
2153 }
2154 }
2155
2156 filesToGroup.clear();
2157 delete groupFile;
2158}
2159#endif //HAVE_FITS
2160
2161// --------------------------------------------------------------------------
2162//
2163//! Implements the StopRun transition.
2164//! Attempts to close the run file.
2165//! @returns
2166//! kSM_WaitingRun if success, kSM_FatalError otherwise
2167int DataLogger::StopRunLogging()
2168{
2169
2170 if (fDebugIsOn)
2171 {
2172 Debug("Stopping Run Logging...");
2173 }
2174 //it may be that dim tries to write a dimInfo at the same time as we're closing files. Prevent this
2175
2176// dim_lock();
2177 for (list<RunNumberType>::const_iterator it=fRunNumber.begin(); it != fRunNumber.end(); it++)
2178 {
2179#ifdef RUN_LOGS
2180 if (!it->logFile->is_open() || !it->reportFile->is_open())
2181#else
2182 if (!it->reportFile->is_open())
2183#endif
2184 return kSM_FatalError;
2185#ifdef RUN_LOGS
2186 it->logFile->close();
2187 Info("Closed: "+it->logName);
2188
2189#endif
2190 it->reportFile->close();
2191 Info("Closed: "+it->reportName);
2192 }
2193
2194#ifdef HAVE_FITS
2195 for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2196 for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2197 {
2198 if (j->second.runFile.IsOpen())
2199 j->second.runFile.Close();
2200 }
2201#endif
2202 NotifyOpenedFile("", 0, fOpenedRunFiles);
2203 if (fNumSubAndFitsIsOn)
2204 fNumSubAndFits->Update();
2205
2206 while (fRunNumber.size() > 0)
2207 {
2208 RemoveOldestRunNumber();
2209 }
2210// dim_unlock();
2211 return kSM_WaitingRun;
2212}
2213// --------------------------------------------------------------------------
2214//
2215//! Implements the Stop and Reset transitions.
2216//! Attempts to close any openned file.
2217//! @returns
2218//! kSM_Ready
2219int DataLogger::GoToReady()
2220{
2221 if (fDebugIsOn)
2222 {
2223 Debug("Going to the Ready state...");
2224 }
2225 if (GetCurrentState() == kSM_Logging || GetCurrentState() == kSM_WaitingRun)
2226 StopRunLogging();
2227
2228 //it may be that dim tries to write a dimInfo while we're closing files. Prevent that
2229 const string baseFileName = CompileFileNameWithPath(fFilePath, "", "");
2230
2231 if (fNightlyReportFile.is_open())
2232 {
2233 fNightlyReportFile.close();
2234 Info("Closed: "+baseFileName+".rep");
2235 }
2236#ifdef HAVE_FITS
2237 for (SubscriptionsListType::iterator i = fServiceSubscriptions.begin(); i != fServiceSubscriptions.end(); i++)
2238 for (map<string, SubscriptionType>::iterator j = i->second.begin(); j != i->second.end(); j++)
2239 {
2240 if (j->second.nightlyFile.IsOpen())
2241 j->second.nightlyFile.Close();
2242 }
2243#endif
2244 if (GetCurrentState() == kSM_Logging ||
2245 GetCurrentState() == kSM_WaitingRun ||
2246 GetCurrentState() == kSM_NightlyOpen)
2247 {
2248 NotifyOpenedFile("", 0, fOpenedNightlyFiles);
2249 if (fNumSubAndFitsIsOn)
2250 fNumSubAndFits->Update();
2251 }
2252#ifdef HAVE_FITS
2253 CreateFitsGrouping(fOpenedNightlyFits, 0);
2254#endif
2255 return kSM_Ready;
2256}
2257
2258// --------------------------------------------------------------------------
2259//
2260//! Implements the transition towards kSM_WaitingRun
2261//! If current state is kSM_Ready, then tries to go to nightlyOpen state first.
2262//! @returns
2263//! kSM_WaitingRun or kSM_BadFolder
2264int DataLogger::NightlyToWaitRun()
2265{
2266 int cState = GetCurrentState();
2267
2268// if (cState == kSM_Logging)
2269// cState = kSM_NightlyOpen;
2270
2271 if (cState == kSM_Ready)
2272 cState = Start();
2273
2274 if (cState != kSM_NightlyOpen)
2275 return GetCurrentState();
2276
2277 if (fDebugIsOn)
2278 {
2279 Debug("Going to Wait Run Number state...");
2280 }
2281 return kSM_WaitingRun;
2282}
2283// --------------------------------------------------------------------------
2284//
2285//! Implements the transition from wait for run number to nightly open
2286//! Does nothing really.
2287//! @returns
2288//! kSM_WaitingRun
2289int DataLogger::BackToNightlyOpen()
2290{
2291 if (GetCurrentState()==kSM_Logging)
2292 StopRunLogging();
2293
2294 if (fDebugIsOn)
2295 {
2296 Debug("Going to NightlyOpen state...");
2297 }
2298 return kSM_NightlyOpen;
2299}
2300// --------------------------------------------------------------------------
2301//
2302//! Setup Logger's configuration from a Configuration object
2303//! @param conf the configuration object that should be used
2304//!
2305int DataLogger::EvalOptions(Configuration& conf)
2306{
2307 fDebugIsOn = conf.Get<bool>("debug");
2308 fFilesStats.SetDebugMode(fDebugIsOn);
2309
2310 //Set the block or allow list
2311 fBlackList.clear();
2312 fWhiteList.clear();
2313
2314 //Adding entries that should ALWAYS be ignored
2315 fBlackList.insert("DATA_LOGGER/MESSAGE");
2316 fBlackList.insert("/SERVICE_LIST");
2317 fBlackList.insert("DIS_DNS/");
2318
2319 //set the black list, white list and the goruping
2320 const vector<string> vec1 = conf.Vec<string>("block");
2321 const vector<string> vec2 = conf.Vec<string>("allow");
2322 const vector<string> vec3 = conf.Vec<string>("group");
2323
2324 fBlackList.insert(vec1.begin(), vec1.end());
2325 fWhiteList.insert(vec2.begin(), vec2.end());
2326 fGrouping.insert( vec3.begin(), vec3.end());
2327
2328 //set the old run numbers timeout delay
2329 if (conf.Has("run-timeout"))
2330 {
2331 const uint32_t timeout = conf.Get<uint32_t>("run-timeout");
2332 if (timeout == 0)
2333 {
2334 Error("Time out delay for old run numbers must not be 0.");
2335 return 1;
2336 }
2337 fRunNumberTimeout = timeout;
2338 }
2339
2340 //configure the run files directory
2341 if (conf.Has("destination-folder"))
2342 {
2343 const string folder = conf.Get<string>("destination-folder");
2344 if (!fFilesStats.SetCurrentFolder(folder))
2345 return 2;
2346
2347 fFilePath = folder;
2348 fFullNightlyLogFileName = CompileFileNameWithPath(fFilePath, "", "log");
2349 if (!OpenTextFile(fNightlyLogFile, fFullNightlyLogFileName))
2350 return 3;
2351
2352 fNightlyLogFile << endl;
2353 NotifyOpenedFile(fFullNightlyLogFileName, 1, fOpenedNightlyFiles);
2354 for (vector<string>::iterator it=backLogBuffer.begin();it!=backLogBuffer.end();it++)
2355 fNightlyLogFile << *it;
2356 }
2357
2358 shouldBackLog = false;
2359 backLogBuffer.clear();
2360
2361 //configure the interval between statistics updates
2362 if (conf.Has("stats-interval"))
2363 fFilesStats.SetUpdateInterval(conf.Get<int16_t>("stats-interval"));
2364
2365 //configure if the filenames service is on or off
2366 fOpenedFilesIsOn = !conf.Get<bool>("no-filename-service");
2367
2368 //configure if the number of subscriptions and fits files is on or off.
2369 fNumSubAndFitsIsOn = !conf.Get<bool>("no-numsubs-service");
2370 //should we open the daily files at startup ?
2371 if (conf.Has("start-daily-files"))
2372 if (conf.Get<bool>("start-daily-files"))
2373 {
2374 fShouldAutoStart = true;
2375 }
2376 return -1;
2377}
2378
2379
2380#include "Main.h"
2381
2382// --------------------------------------------------------------------------
2383template<class T>
2384int RunShell(Configuration &conf)
2385{
2386 return Main::execute<T, DataLogger>(conf, true);
2387}
2388
2389/*
2390 Extract usage clause(s) [if any] for SYNOPSIS.
2391 Translators: "Usage" and "or" here are patterns (regular expressions) which
2392 are used to match the usage synopsis in program output. An example from cp
2393 (GNU coreutils) which contains both strings:
2394 Usage: cp [OPTION]... [-T] SOURCE DEST
2395 or: cp [OPTION]... SOURCE... DIRECTORY
2396 or: cp [OPTION]... -t DIRECTORY SOURCE...
2397 */
2398void PrintUsage()
2399{
2400 cout << "\n"
2401 "The data logger connects to all available Dim services and "
2402 "writes them to ascii and fits files.\n"
2403 "\n"
2404 "The default is that the program is started without user interaction. "
2405 "All actions are supposed to arrive as DimCommands. Using the -c "
2406 "option, a local shell can be initialized. With h or help a short "
2407 "help message about the usage can be brought to the screen.\n"
2408 "\n"
2409 "Usage: datalogger [-c type] [OPTIONS]\n"
2410 " or: datalogger [OPTIONS]\n";
2411 cout << endl;
2412
2413}
2414// --------------------------------------------------------------------------
2415void PrintHelp()
2416{
2417 /* Additional help text which is printed after the configuration
2418 options goes here */
2419 cout <<
2420 "\n"
2421 "If the allow list has any element, only the servers and/or services "
2422 "specified in the list will be used for subscription. The black list "
2423 "will disable service subscription and has higher priority than the "
2424 "allow list. If the allow list is not present by default all services "
2425 "will be subscribed."
2426 "\n"
2427 "For example, block=DIS_DNS/ will skip all the services offered by "
2428 "the DIS_DNS server, while block=/SERVICE_LIST will skip all the "
2429 "SERVICE_LIST services offered by any server and DIS_DNS/SERVICE_LIST "
2430 "will skip DIS_DNS/SERVICE_LIST.\n"
2431 << endl;
2432
2433 Main::PrintHelp<DataLogger>();
2434}
2435
2436// --------------------------------------------------------------------------
2437void SetupConfiguration(Configuration &conf)
2438{
2439 po::options_description configs("DataLogger options");
2440 configs.add_options()
2441 ("block,b", vars<string>(), "Black-list to block services")
2442 ("allow,a", vars<string>(), "White-list to only allowe certain services")
2443 ("debug,d", po_bool(), "Debug mode. Print clear text of received service reports.")
2444 ("group,g", vars<string>(), "Grouping of services into a single run-Fits")
2445 ("run-timeout", var<uint32_t>(), "Time out delay for old run numbers in milliseconds.")
2446 ("destination-folder", var<string>(), "Base path for the nightly and run files")
2447 ("stats-interval", var<int16_t>(), "Interval in milliseconds for write statistics update")
2448 ("no-filename-service", po_bool(), "Disable update of filename service")
2449 ("no-numsubs-service", po_bool(), "Disable update of number-of-subscriptions service")
2450 ("start-daily-files", po_bool(), "Starts the logger in DailyFileOpen instead of Ready")
2451 ;
2452
2453 conf.AddOptions(configs);
2454}
2455// --------------------------------------------------------------------------
2456int main(int argc, const char* argv[])
2457{
2458 Configuration conf(argv[0]);
2459 conf.SetPrintUsage(PrintUsage);
2460 Main::SetupConfiguration(conf);
2461 SetupConfiguration(conf);
2462
2463 if (!conf.DoParse(argc, argv, PrintHelp))
2464 return 127;
2465
2466// try
2467 {
2468 // No console access at all
2469 if (!conf.Has("console"))
2470 return RunShell<LocalStream>(conf);
2471
2472 // Console access w/ and w/o Dim
2473 if (conf.Get<int>("console")==0)
2474 return RunShell<LocalShell>(conf);
2475 else
2476 return RunShell<LocalConsole>(conf);
2477 }
2478/* catch (exception& e)
2479 {
2480 cerr << "Exception: " << e.what() << endl;
2481 return -1;
2482 }*/
2483
2484 return 0;
2485}
Note: See TracBrowser for help on using the repository browser.