Index: /trunk/FACT++/src/datalogger.cc =================================================================== --- /trunk/FACT++/src/datalogger.cc (revision 11462) +++ /trunk/FACT++/src/datalogger.cc (revision 11463) @@ -59,4 +59,5 @@ #include "DimNetwork.h" + #ifdef HAVE_FITS #include "Fits.h" @@ -64,10 +65,4 @@ //Dim structures -///Distributes the writing statistics -//struct DataLoggerStats { -// long sizeWritten; -// long freeSpace; -// long writingRate; -//}; ///distributes the number of opened subscriptions and fits files struct NumSubAndFitsType { @@ -283,20 +278,9 @@ void ServicesMonitoring(); inline void NotifyOpenedFile(const string &name, int type, DimDescribedService* service); - ///variables for computing statistics -// DataLoggerStats fStatVar; - ///stores the size of each file that is or was open -// map fFileSizesMap; - ///total size of the opened files BEFORE they were opened by the logger -// long fBaseSizeNightly; -// long fPreviousSize; -// long fBaseSizeRun; ///Service for opened files DimDescribedService* fOpenedNightlyFiles; DimDescribedService* fOpenedRunFiles; DimDescribedService* fNumSubAndFits; -// DimDescribedService* fStatsMonitoring; NumSubAndFitsType fNumSubAndFitsData; - ///Small function for calculating the total size written so far -// bool calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting); /*************************************************** @@ -358,8 +342,4 @@ ///Open a text file and checks for ofstream status bool OpenTextFilePlease(ofstream& stream, const string& name); - ///Check if a dir is . and returns the actual string corresponding to . -// string CheckIfDirIsDot(const string& dir); - ///Remembers the size of newly opened files. for statistic purposes -// bool RememberFileOrigSizePlease(string& fileName, bool nightly); ///Checks if the input osftream is in error state, and if so close it. bool CheckForOfstreamError(ofstream& out, bool isDailyStream); @@ -369,6 +349,4 @@ ///Checks if a given path exist bool DoesPathExist(string path); - ///Check if the statistics service should be updated, and if so, do it -// void UpdateStatisticsService(); ///Check if old run numbers can be trimmed, and if so, do it void TrimOldRunNumbers(); @@ -384,10 +362,39 @@ ///Remove all the services associated with a given server void RemoveAllServices(const string&); - ///Remove a server and all of its services - //void RemoveServer(const std::string &s); - ///Remove all the servers and all subscriptions - //void RemoveAllServers(); + /*************************************************** + * Overwritten from MessageImp + ***************************************************/ + vector backLogBuffer; + bool shouldBackLog; +public: + int Write(const Time &time, const std::string &txt, int qos=kMessage); + }; //DataLogger +// -------------------------------------------------------------------------- +// +//! Overwritten write function. This way we directly log the datalogger's messages, without going through dim's dns, +//! thus increasing robustness. +//! @param time: see MessageImp class param +//! @param txt: see MessageImp class param +//! @param qos: see MessageImp class param +//! @return see MessageImp class param +// +int DataLogger::Write(const Time&time, const std::string& txt, int qos) +{ + if (fNightlyLogFile.is_open()) + { + MessageImp mimp(fNightlyLogFile); + mimp.Write(time, txt, qos); + } + else if (shouldBackLog) + { + ostringstream str; + MessageImp mimp(str); + mimp.Write(time, txt, qos); + backLogBuffer.push_back(str.str()); + } + return StateMachineDim::Write(time, txt, qos); +} // -------------------------------------------------------------------------- // @@ -489,5 +496,4 @@ } -// list[service].dimInfo = shared_ptr(SubscribeToPlease(server, service)); list[service].dimInfo.reset(SubscribeToPlease(server, service)); list[service].server = server; @@ -544,27 +550,4 @@ } } -/* - // -------------------------------------------------------------------------- -// -//! Remove a given server -//! @param s the server to be removed -// -void DataLogger::RemoveServer(const std::string &s) -{ - if (fServiceSubscriptions.erase(s) != 1) - { - Error("The server " + s + " to be removed was not in the servers list"); - } - DimServiceInfoList::RemoveServer(s); -} -// -------------------------------------------------------------------------- -// -//! Remove all the server and associated subscriptions -// -void DataLogger::RemoveAllServers() -{ - fServiceSubscriptions.clear(); - DimServiceInfoList::RemoveAllServers(); - }*/ // -------------------------------------------------------------------------- @@ -588,23 +571,4 @@ return false; } -// -------------------------------------------------------------------------- -// -//! Checks the size on disk of a given size, and remembers it in the relevant member variable -//! @param fileName the file for which the size on disk should be retrieved -//! @param nightly whether this is a run or nightly file, so that its size is added to the correct member variable -// -//bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly) -//{ - //get the size of the file we're about to open -// if (fFileSizesMap.find(fileName) != fFileSizesMap.end()) -// return false; - -// if (nightly) -// fBaseSizeNightly += GetFileSize(fileName); -// else -// fBaseSizeRun += GetFileSize(fileName); -// fFileSizesMap[fileName] = 0; -// return true;// -//} // -------------------------------------------------------------------------- @@ -617,6 +581,7 @@ { errno = 0; + if (!stream.is_open()) stream.open(name.c_str(), ios_base::out | ios_base::app); - if (!stream) + if (!stream.is_open()) { ostringstream str; @@ -817,84 +782,4 @@ // -------------------------------------------------------------------------- // -//! Calculate the total number of written bytes since the logger was started -//! @param statVar the data structure that should be updated -//! @param shouldWarn whether or not error messages should be outputted -//! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information -// -/* -bool DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting) -{ -#ifdef HAVE_FITS - if (isPrinting) - { - ostringstream str; - str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:"; - Message(str); - } - - ///TODO the grouping file is dealt with several times. This should not be a problem but well, better to fix it I guess. - for (SubscriptionsListType::const_iterator x=fServiceSubscriptions.begin(); - x!=fServiceSubscriptions.end(); x++) - { - for (map::const_iterator y=x->second.begin(); - y!=x->second.end(); y++) - { - if (y->second.runFile.IsOpen()) - { - fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); - if (isPrinting) - Message("-> "+y->second.runFile.fFileName); - } - if (y->second.nightlyFile.IsOpen()) - { - fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); - if (isPrinting) - Message("-> "+y->second.nightlyFile.fFileName); - } - } - } -#else - if (isPrinting) - Message("FITS output disabled at compilation"); -#endif - //gather log and report files sizes on disk - if (fNightlyLogFile.is_open()) - fFileSizesMap[fFullNightlyLogFileName] = GetFileSize(fFullNightlyLogFileName); - if (fNightlyReportFile.is_open()) - fFileSizesMap[fFullNightlyReportFileName] = GetFileSize(fFullNightlyReportFileName); - for (list::iterator it = fRunNumber.begin(); it != fRunNumber.end(); it++) - { - if (it->reportFile->is_open()) - fFileSizesMap[it->reportName] = GetFileSize(it->reportName); -#ifdef RUN_LOGS - if (it->logFile->is_open()) - fFileSizesMap[it->logName] = GetFileSize(it->logName); -#endif - } - - bool shouldWarn = false; - struct statvfs vfs; - if (!statvfs(fNightlyFilePath.c_str(), &vfs)) - statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; - else - { - ostringstream str; - str << "Unable to retrieve stats for " << fNightlyFilePath << ". Reason: " << strerror(errno) << " [" << errno << "]"; - if (!shouldWarn) - Error(str); - statVar.freeSpace = -1; - } - //sum up all the file sizes. past and present - statVar.sizeWritten = 0; - for (map::const_iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) - statVar.sizeWritten += it->second; - statVar.sizeWritten -= fBaseSizeNightly; - statVar.sizeWritten -= fBaseSizeRun; - - return shouldWarn; -} -*/ -// -------------------------------------------------------------------------- -// //! Default constructor. The name of the machine is given DATA_LOGGER //! and the state is set to kSM_Ready at the end of the function. @@ -905,4 +790,5 @@ fFilesStats("DATA_LOGGER", *this) { + shouldBackLog = true; //initialize member data fNightlyFilePath = "."; @@ -989,7 +875,6 @@ //services parameters - fDebugIsOn = false; -// fStatsPeriodDuration = 1.0f; - fOpenedFilesIsOn = true; + fDebugIsOn = false; + fOpenedFilesIsOn = true; fNumSubAndFitsIsOn = true; @@ -1022,22 +907,6 @@ fDestructing = false; - //start the monitoring service -// fStatVar.sizeWritten = 0; -// fStatVar.freeSpace = 0; -// fStatVar.writingRate = 0; -// fPreviousStatsUpdateTime = Time().Mjd(); fPreviousOldRunNumberCheck = Time().Mjd(); -// fPreviousSize = 0; - -// struct statvfs vfs; -// if (!statvfs(fNightlyFilePath.c_str(), &vfs)) -// fStatVar.freeSpace = vfs.f_bsize*vfs.f_bavail; -// else -// fStatVar.freeSpace = -1; - -// fStatsMonitoring = new DimDescribedService(GetName() + "/STATS", "X:3", fStatVar, "Add description here"); - -// fBaseSizeNightly = 0; -// fBaseSizeRun = 0; + fDailyFileDayChangedAlready = true; fRunNumberTimeout = 60; //default run-timeout set to 1 minute @@ -1057,18 +926,18 @@ Debug("DataLogger destruction starts"); - //this boolean should not be required anymore, because it + //this boolean should not be required anymore fDestructing = true; //first, let's backup the datalogger/message service subscription - shared_ptr messageBackup; - const SubscriptionsListType::iterator x = fServiceSubscriptions.find("DATA_LOGGER"); - if (x != fServiceSubscriptions.end()) - { - const map::iterator y = x->second.find("MESSAGE"); - if (y != x->second.end()) - messageBackup = y->second.dimInfo; - } - - dim_lock(); +// shared_ptr messageBackup; +// const SubscriptionsListType::iterator x = fServiceSubscriptions.find("DATA_LOGGER"); +// if (x != fServiceSubscriptions.end()) +// { +// const map::iterator y = x->second.find("MESSAGE"); +// if (y != x->second.end()) +// messageBackup = y->second.dimInfo; +// } + +// dim_lock(); //now clear the services subscriptions @@ -1083,5 +952,5 @@ GoToReadyPlease(); - dim_unlock(); +// dim_unlock(); Info("Will soon close the daily log file"); @@ -1092,12 +961,12 @@ //release message service before closing nightly log file - if (messageBackup) - messageBackup.reset(); +// if (messageBackup) + // messageBackup.reset(); if (fNightlyLogFile.is_open())//this file is the only one that has not been closed by GoToReadyPlease { - dim_lock(); +// dim_lock(); fNightlyLogFile.close(); - dim_unlock(); +// dim_unlock(); } @@ -1105,36 +974,5 @@ Debug("DataLogger desctruction ends"); } -// -------------------------------------------------------------------------- -// -//! checks if the statistic service should be updated, and if so, do it -// -/*void DataLogger::UpdateStatisticsService() -{ - //update the fits files sizes - const Time cTime = Time(); - - if ((fStatsPeriodDuration == 0) || ((cTime - fPreviousStatsUpdateTime).total_seconds() < fStatsPeriodDuration)) - return; - - calculateTotalSizeWritten(fStatVar, false); - fStatVar.writingRate = (fStatVar.sizeWritten - fPreviousSize)/((cTime - fPreviousStatsUpdateTime).total_seconds()); - fPreviousSize = fStatVar.sizeWritten; - fPreviousStatsUpdateTime = cTime; - //update the service. No need to check if data has been written, because some must have been, otherwise we would not have hit this piece of code - if (fStatVar.writingRate < 0) - fStatVar.writingRate = 0; - if (fStatVar.sizeWritten < 0) - fStatVar.sizeWritten = 0; - fStatsMonitoring->updateService(); - - if(fDebugIsOn) - { - ostringstream str; - str << "Size written: " << fStatVar.sizeWritten/1000 << " kB; writing rate: "; - str << fStatVar.writingRate/1000 << " kB/s; free space: "; - str << fStatVar.freeSpace/(1000*1000) << " MB"; - Debug(str); - } -}*/ + // -------------------------------------------------------------------------- // @@ -1164,7 +1002,4 @@ if (I==NULL) return; - - //it may happen that we try to write a dimInfo while closing files. Prevent that with a dim_lock() -// dim_lock(); //check if the service pointer corresponds to something that we subscribed to @@ -1187,10 +1022,8 @@ { DimServiceInfoList::infoHandler(); -// dim_unlock(); return; } if (I->getSize() <= 0 || I->getData()==NULL) { -// dim_unlock(); return; } @@ -1207,5 +1040,4 @@ if (I->getTimestamp() == 0) { -// dim_unlock(); return; } @@ -1217,8 +1049,4 @@ ReportPlease(I, y->second); - -// dim_unlock(); - //update the fits files sizes - // UpdateStatisticsService(); //remove old run numbers @@ -1281,7 +1109,5 @@ #ifdef RUN_LOGS fFilesStats.FileOpened(run.logName); -// RememberFileOrigSizePlease(run.logName, false); #endif -// RememberFileOrigSizePlease(run.reportName, false); fFilesStats.FileOpened(run.reportName); //TODO this notification scheme might be messed up now.... fix it ! @@ -1746,32 +1572,4 @@ int DataLogger::SetStatsPeriod(const Event& evt) { -/* const float backupDuration = fStatsPeriodDuration; - - fStatsPeriodDuration = evt.GetFloat(); - - if (fStatsPeriodDuration < 0) - { - Error("Statistics period duration should be greater than zero. Discarding provided value."); - fStatsPeriodDuration = backupDuration; - return GetCurrentState(); - } - if (!finite(fStatsPeriodDuration))// != fStatsPeriodDuration) - { - Error("Provided duration does not appear to be a valid float. Discarding it."); - fStatsPeriodDuration = backupDuration; - return GetCurrentState(); - } - if (backupDuration == fStatsPeriodDuration) - Warn("Statistics period not modified. Supplied value already in use."); - - if (fStatsPeriodDuration == 0.0f) - Message("Statistics are now OFF"); - else - { - ostringstream str; - str << "Statistics period is now " << fStatsPeriodDuration << " seconds"; - Message(str); - } -*/ fFilesStats.SetUpdateInterval(evt.GetShort()); return GetCurrentState(); @@ -1987,11 +1785,5 @@ return kSM_BadNightlyConfig; } - //get the size of the newly opened file. -// fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName); -// fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName); -// fFileSizesMap.clear(); -// fBaseSizeRun = 0; -// fPreviousSize = 0; -// fFilesStats.Reset(); + fFilesStats.FileOpened(fFullNightlyLogFileName); fFilesStats.FileOpened(fFullNightlyReportFileName); @@ -2053,5 +1845,4 @@ //get the size of the file we're about to open if (fFilesStats.FileOpened(partialName)) -// if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update) fOpenedNightlyFits[fileNameOnly].push_back(serviceName); @@ -2059,5 +1850,4 @@ { GoToRunWriteErrorState(); - //SetCurrentState(kSM_WriteError); return; } @@ -2090,5 +1880,4 @@ //get the size of the file we're about to open if (fFilesStats.FileOpened(partialName)) -// if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update) cRunNumber->openedFits[fileNameOnly].push_back(serviceName); else @@ -2109,12 +1898,11 @@ str << "Open FITS file " << partialName << ": " << e.message(); Error(str); - cRunNumber->runFitsFile = shared_ptr();//NULL; + cRunNumber->runFitsFile = shared_ptr(); GoToRunWriteErrorState(); - //SetCurrentState(kSM_WriteError); return; } const string baseFileName = CompileFileNameWithPath(fRunFilePath, sub.runNumber, "", ""); - NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles);// + '_' + serviceName, 4); + NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles); if (hasGrouping) @@ -2123,5 +1911,4 @@ { GoToRunWriteErrorState(); - //SetCurrentState(kSM_WriteError); return; } @@ -2132,5 +1919,4 @@ { GoToRunWriteErrorState(); - //SetCurrentState(kSM_WriteError); return; } @@ -2216,5 +2002,4 @@ GoToNightlyWriteErrorState(); return; - //SetCurrentState(kSM_WriteError); } } @@ -2227,5 +2012,4 @@ GoToRunWriteErrorState(); return; - //SetCurrentState(kSM_WriteError); } } @@ -2481,12 +2265,6 @@ StopRunPlease(); //it may be that dim tries to write a dimInfo while we're closing files. Prevent that -// dim_lock(); const string baseFileName = CompileFileNameWithPath(fNightlyFilePath, "", ""); -//we should log everything that we can, even if in Ready state. thus removing this closing (done only in the destructor or at day-change) -// if (fNightlyLogFile.is_open() && !fDestructing)//in case we're destructing the datalogger, wait before closing the log file. -// { -// fNightlyLogFile.close(); -// Info("Closed: "+baseFileName+".log"); -// } + if (fNightlyReportFile.is_open()) { @@ -2513,5 +2291,4 @@ CreateFitsGrouping(fOpenedNightlyFits, 0); #endif -// dim_unlock(); return kSM_Ready; } @@ -2567,5 +2344,5 @@ //Adding entries that should ALWAYS be ignored - //fBlackList.insert("DATA_LOGGER/"); + fBlackList.insert("DATA_LOGGER/MESSAGE"); fBlackList.insert("/SERVICE_LIST"); fBlackList.insert("DIS_DNS/"); @@ -2604,5 +2381,14 @@ if (!OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName)) SetCurrentState(kSM_BadNightlyConfig); - } + else + { + NotifyOpenedFile(fFullNightlyLogFileName, 1, fOpenedNightlyFiles); + for (vector::iterator it=backLogBuffer.begin();it!=backLogBuffer.end();it++) + fNightlyLogFile << *it << endl; + } + } + + shouldBackLog = false; + backLogBuffer.clear(); //configure the interval between statistics updates @@ -2619,40 +2405,7 @@ } -// -------------------------------------------------------------------------- -/* - int RunDim(Configuration &conf) -{ - WindowLog wout; - - ReadlineColor::PrintBootMsg(wout, conf.GetName(), false); - - //log.SetWindow(stdscr); - if (conf.Has("log")) - if (!wout.OpenLogFile(conf.Get("log"))) - wout << kRed << "ERROR - Couldn't open log-file " << conf.Get("log") << ": " << strerror(errno) << endl; - - // Start io_service.Run to use the StateMachineImp::Run() loop - // Start io_service.run to only use the commandHandler command detaching - DataLogger logger(wout); - if (!logger.EvalConfiguration(conf)) - return -1; - - logger.Run(true); - - return 0; -} -*/ -// -------------------------------------------------------------------------- #include "Main.h" -/* -void RunThread(DataLogger* logger) -{ - // This is necessary so that the StateMachine Thread can signal the - // Readline thread to exit - logger->Run(true); - Readline::Stop(); -} -*/ + // -------------------------------------------------------------------------- template @@ -2660,52 +2413,4 @@ { return Main(conf, true); -/* - static T shell(conf.GetName().c_str(), conf.Get("console")!=1); - - WindowLog &win = shell.GetStreamIn(); - WindowLog &wout = shell.GetStreamOut(); - - if (conf.Has("log")) - if (!wout.OpenLogFile(conf.Get("log"))) - win << kRed << "ERROR - Couldn't open log-file " << conf.Get("log") << ": " << strerror(errno) << endl; - - DataLogger logger(wout); - - if (!logger.EvalConfiguration(conf)) - return -1; - - shell.SetReceiver(logger); - - boost::thread t(boost::bind(RunThread, &logger)); - - if (conf.Has("cmd")) - { - const vector v = conf.Get>("cmd"); - for (vector::const_iterator it=v.begin(); it!=v.end(); it++) - shell.ProcessLine(*it); - } - - if (conf.Has("exec")) - { - const vector v = conf.Get>("exec"); - for (vector::const_iterator it=v.begin(); it!=v.end(); it++) - shell.Execute(*it); - } - - if (conf.Get("quit")) - shell.Stop(); - - - shell.Run(); // Run the shell - - logger.Stop(); - - //Wait until the StateMachine has finished its thread - //before returning and destroyinng the dim objects which might - //still be in use. - t.join(); - - return 0; -*/ }