Index: /trunk/FACT++/src/FilesStatisticsService.cc =================================================================== --- /trunk/FACT++/src/FilesStatisticsService.cc (revision 11290) +++ /trunk/FACT++/src/FilesStatisticsService.cc (revision 11290) @@ -0,0 +1,238 @@ +#include "FilesStatisticsService.h" +#include //for getting disk free space +#include //for getting files sizes +#include "Time.h" +// -------------------------------------------------------------------------- +// +//! Default constructor. Should NOT be used otherwise the service name will be wrong +// +//FilesStatisticsService::FilesStatisticsService() +//{ + // FilesStatisticsService("YouDidntCallTheProperConstructor"); +//} +// -------------------------------------------------------------------------- +// +//! Constructor with correct service name. The state machine using this object should give it +//! its own name as a parameter +//! @param serverName the name of the server which created this object +// +FilesStatisticsService::FilesStatisticsService(const string& serverName, MessageImp* mess) : fServerName(serverName), + fCurrentFolder("."), + fContinueStats(true), + fBaseSize(0), + fPeriodDuration(1), + fMess(mess) +{ + fStats.sizeWritten = 0; + fStats.freeSpace = 0; + fStats.writingRate = 0; + fService = new DimDescribedService(serverName + "/STATS", "X:3", fStats, "Statistics about size written"); + fThread = boost::thread(boost::bind(&FilesStatisticsService::UpdateService, this)); +} +// -------------------------------------------------------------------------- +// +//! Default destructor. +// +FilesStatisticsService::~FilesStatisticsService() +{ + fContinueStats = false; +//WARNING: DO NOT DO JOIN OTHERWISE THE DESTRUCTOR BLOCKS +// fThread.join(); + delete fService; +} +// -------------------------------------------------------------------------- +// +//! Retrieves the free space of the current base path +//! @return the available free space on disk, in bytes +// +long FilesStatisticsService::GetFreeSpace() +{ + struct statvfs vfs; + if (statvfs(fCurrentFolder.c_str(), &vfs)) + return -1; + return vfs.f_bsize*vfs.f_bavail; +} +// -------------------------------------------------------------------------- +// +//! Retrieves the size on disk of a given file, in bytes +//! @param file the filename for which the size should be retrieved +//! @return the size of the file, in bytes +// +long FilesStatisticsService::GetFileSizeOnDisk(const string& file) +{ + errno = 0; + struct stat st; + if (!stat(file.c_str(), &st)) + return st.st_size; + + if (errno != 0 && errno != 2)//ignoring error #2: no such file or directory is not an error for new files + { + ostringstream str; + str << "Unable to stat " << file << ". Reason: " << strerror(errno) << " [" << errno << "]"; + fMess->Error(str); + } + + return -1; +} +// -------------------------------------------------------------------------- +// +//! Sets the current folder +//! @param folder the path to the folder +// +bool FilesStatisticsService::SetCurrentFolder(string& folder) +{ + fMutex.lock(); + fCurrentFolder = folder; + + fStats.freeSpace = GetFreeSpace(); + if (fStats.freeSpace == -1) + { + fMess->Error("Could not stat the given folder. Ignoring it"); + fMutex.unlock(); + return false; + } + fMutex.unlock(); + return true; +} +// -------------------------------------------------------------------------- +// +//! Updates the service. This is the function executed by the thread +// +void FilesStatisticsService::UpdateService() +{ + Time cTime = Time(); + Time previousTime; + fStats.freeSpace = GetFreeSpace(); + fService->updateService(); + sleep(1); + while (fContinueStats) + { + if (fPeriodDuration == 0.0f) + { + sleep(0.1f); + continue; + } + sleep(fPeriodDuration); + fMutex.lock(); + previousTime = cTime; + cTime = Time(); + fStats.freeSpace = GetFreeSpace(); + //gather the size of opened files. + long previousSize = fStats.sizeWritten; + fStats.sizeWritten = 0; + for (set::const_iterator it = fOpenedFiles.begin(); it != fOpenedFiles.end(); it++) + fStats.sizeWritten += GetFileSizeOnDisk(*it); + fStats.sizeWritten -= fBaseSize; + long timeElapsed = (cTime - previousTime).total_seconds(); + if (timeElapsed != 0) + fStats.writingRate = (fStats.sizeWritten - previousSize)/timeElapsed; + else + fStats.writingRate = 0; + fService->updateService(); + fMutex.unlock(); + if (fDebug) + { + ostringstream str; + str << "Size written: " << fStats.sizeWritten/1024 << " kB; writing rate: "; + str << fStats.writingRate/1024 << " kB/s; free space: "; + str << fStats.freeSpace/(1024*1024) << " MB"; + fMess->Debug(str); + } + } +} +// -------------------------------------------------------------------------- +// +//! Let the object know that a new file has been opened +//! @param fileName the full name of the file newly opened +//! @return whether this file could be stated or not +// +bool FilesStatisticsService::FileOpened(const string& fileName) +{ + if (fOpenedFiles.find(fileName) != fOpenedFiles.end()) + return false; + fMutex.lock(); + //Add a newly opened file, and remember its original size + long newSize = GetFileSizeOnDisk(fileName); + if (newSize != -1) + { + fBaseSize += newSize; + fOpenedFiles.insert(fileName); + } + else + { + fMess->Error("Could not stat file name: " + fileName); + fMutex.unlock(); + return false; + } + fMutex.unlock(); + return true; +} +// -------------------------------------------------------------------------- +// +//! Set the debug mode on and off +//! @param debug the new mode (true or false) +// +void FilesStatisticsService::SetDebugMode(bool debug) +{ + if (fDebug == debug) + { + fMess->Error("Debug mode already in the asked state"); + return; + } + fDebug = debug; + if (fDebug) + { + fMess->Debug("Debug mode is now on"); + } +} +// -------------------------------------------------------------------------- +// +//! Set the update of the service interval +//! @param duration the duration between two services update, in second +// +void FilesStatisticsService::SetStatPeriod(float duration) +{ + if (duration < 0) + { + fMess->Error("Statistics period duration should be greater than zero. Discarding"); + return; + } + if (!finite(duration)) + { + fMess->Error("Provided duration does not appear to be a valid float. Discarding"); + return; + } + if (duration == fPeriodDuration) + fMess->Warn("Statistics period not modified. Supplied value already in use"); + if (duration == 0) + fMess->Message("Statistics are now OFF"); + else + { + ostringstream str; + str << "Statistics period is now " << duration << " seconds"; + fMess->Message(str); + } + fPeriodDuration = duration; +} +// -------------------------------------------------------------------------- +// +//! Retrieves the latest calculated size written and free disk space +//! @param the FileStatisticsData data structure to be filled in +// +void FilesStatisticsService::GetTotalSizeWritten(FileStatisticsData& data) +{ + data.sizeWritten = fStats.sizeWritten; + data.writingRate = fStats.writingRate; + data.freeSpace = fStats.freeSpace; +} +// -------------------------------------------------------------------------- +// +//! Resets the files statistics object. Basically sets all counters to zero +// +void FilesStatisticsService::Reset() +{ + fStats.sizeWritten = 0; + fStats.writingRate = 0; + fBaseSize = 0; + fOpenedFiles.clear(); +} Index: /trunk/FACT++/src/FilesStatisticsService.h =================================================================== --- /trunk/FACT++/src/FilesStatisticsService.h (revision 11290) +++ /trunk/FACT++/src/FilesStatisticsService.h (revision 11290) @@ -0,0 +1,57 @@ +//************************************************************************************* +/** @class FilesStatisticsService + * + * @brief provides a statistics service telling the free space on disk and the total size written so far + * + */ +//************************************************************************************* +#ifndef FILESSTATISTICSSERVICE_H_ +#define FILESSTATISTICSSERVICE_H_ + +#include "MessageDim.h" +#include +#include +#include +#include + +using namespace std; + +struct FileStatisticsData { + long sizeWritten; + long freeSpace; + long writingRate; +}; +class FilesStatisticsService +{ +private: + string fServerName; + string fCurrentFolder; + boost::thread fThread; + FileStatisticsData fStats; + mutex fMutex; + DimDescribedService* fService; + bool fContinueStats; + bool fDebug; + void UpdateService(); + long GetFreeSpace(); + long GetFileSizeOnDisk(const string& file); + long fBaseSize; + set fOpenedFiles; + float fPeriodDuration; + MessageImp* fMess; +public: + ///Default constructor +// FilesStatisticsService(); + FilesStatisticsService(const string& serverName, MessageImp* mess); + ///Default destructor + ~FilesStatisticsService(); + ///Configures that current folder where files are written to + bool SetCurrentFolder(string& folder); + bool FileOpened(const string& fileName); + void SetDebugMode(bool); + void SetStatPeriod(float duration); + void GetTotalSizeWritten(FileStatisticsData& data); + void Reset(); +}; + +#endif /* FILESSTATISTICSSERVICE_H_ */ Index: /trunk/FACT++/src/Fits.cc =================================================================== --- /trunk/FACT++/src/Fits.cc (revision 11289) +++ /trunk/FACT++/src/Fits.cc (revision 11290) @@ -347,5 +347,4 @@ // fFile = NULL; WriteSingleHeaderKey("TSTOP", fEndMjD, "Time of the last receied data"); -// fFile->flush(); delete fFile; Index: /trunk/FACT++/src/datalogger.cc =================================================================== --- /trunk/FACT++/src/datalogger.cc (revision 11289) +++ /trunk/FACT++/src/datalogger.cc (revision 11290) @@ -37,4 +37,6 @@ } \enddot + + For questions or bug report, please contact Etienne Lyard (etienne.lyard@unige.ch) or Thomas Bretz. */ //**************************************************************** @@ -67,11 +69,13 @@ #endif +#include "FilesStatisticsService.h" + //Dim structures ///Distributes the writing statistics -struct DataLoggerStats { - long sizeWritten; - long freeSpace; - long writingRate; -}; +//struct DataLoggerStats { +// long sizeWritten; +// long freeSpace; +// long writingRate; +//}; ///distributes the number of opened subscriptions and fits files struct NumSubAndFitsType { @@ -230,9 +234,9 @@ string fFullNightlyReportFileName; ///variable to track when the statistic were last calculated - Time fPreviousStatsUpdateTime; +// Time fPreviousStatsUpdateTime; Time fPreviousOldRunNumberCheck; ///boolean to know whether we should close and reopen daily files or not bool fDailyFileDayChangedAlready; - + FilesStatisticsService fFilesStats; private: /*************************************************** @@ -269,4 +273,6 @@ ///from NightlyOpen to waiting transition int NightlyToWaitRunPlease(); + ///from wait for run number to nightly open + int BackToNightlyOpenPlease(); #ifdef HAVE_FITS ///Open fits files @@ -285,19 +291,19 @@ inline void NotifyOpenedFile(const string &name, int type, DimDescribedService* service); ///variables for computing statistics - DataLoggerStats fStatVar; +// DataLoggerStats fStatVar; ///stores the size of each file that is or was open - map fFileSizesMap; +// map fFileSizesMap; ///total size of the opened files BEFORE they were opened by the logger - long fBaseSizeNightly; - long fPreviousSize; - long fBaseSizeRun; +// long fBaseSizeNightly; +// long fPreviousSize; +// long fBaseSizeRun; ///Service for opened files DimDescribedService* fOpenedNightlyFiles; DimDescribedService* fOpenedRunFiles; DimDescribedService* fNumSubAndFits; - DimDescribedService* fStatsMonitoring; +// DimDescribedService* fStatsMonitoring; NumSubAndFitsType fNumSubAndFitsData; ///Small function for calculating the total size written so far - bool calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting); +// bool calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting); /*************************************************** @@ -362,5 +368,5 @@ // string CheckIfDirIsDot(const string& dir); ///Remembers the size of newly opened files. for statistic purposes - bool RememberFileOrigSizePlease(string& fileName, bool nightly); +// bool RememberFileOrigSizePlease(string& fileName, bool nightly); ///Checks if the input osftream is in error state, and if so close it. bool CheckForOfstreamError(ofstream& out, bool isDailyStream); @@ -371,5 +377,5 @@ bool DoesPathExist(string path); ///Check if the statistics service should be updated, and if so, do it - void UpdateStatisticsService(); +// void UpdateStatisticsService(); ///Check if old run numbers can be trimmed, and if so, do it void TrimOldRunNumbers(); @@ -592,17 +598,17 @@ //! @param nightly whether this is a run or nightly file, so that its size is added to the correct member variable // -bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly) -{ +//bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly) +//{ //get the size of the file we're about to open - if (fFileSizesMap.find(fileName) != fFileSizesMap.end()) - return false; - - if (nightly) - fBaseSizeNightly += GetFileSize(fileName); - else - fBaseSizeRun += GetFileSize(fileName); - fFileSizesMap[fileName] = 0; - return true; -} +// if (fFileSizesMap.find(fileName) != fFileSizesMap.end()) +// return false; + +// if (nightly) +// fBaseSizeNightly += GetFileSize(fileName); +// else +// fBaseSizeRun += GetFileSize(fileName); +// fFileSizesMap[fileName] = 0; +// return true;// +//} // -------------------------------------------------------------------------- @@ -820,4 +826,5 @@ //! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information // +/* bool DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting) { @@ -891,5 +898,5 @@ return shouldWarn; } - +*/ // -------------------------------------------------------------------------- // @@ -899,5 +906,6 @@ //!Setup the allows states, configs and transitions for the data logger // -DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER") +DataLogger::DataLogger(ostream &out) : StateMachineDim(out, "DATA_LOGGER"), + fFilesStats("DATA_LOGGER", this) { //initialize member data @@ -935,7 +943,11 @@ ("Transition to exit error states. Closes the any open file."); - AddEvent(kSM_WaitingRun, "WAIT_FOR_RUN_NUMBER", kSM_NightlyOpen) + AddEvent(kSM_WaitingRun, "WAIT_FOR_RUN_NUMBER", kSM_NightlyOpen, kSM_Ready) (boost::bind(&DataLogger::NightlyToWaitRunPlease, this)) ("Go to waiting for run number state. In this state with any received run-number a new file is opened."); + + AddEvent(kSM_NightlyOpen, "BACK_TO_NIGHTLY_OPEN", kSM_WaitingRun) + (boost::bind(&DataLogger::BackToNightlyOpenPlease, this)) + ("Go from the wait for run to nightly open state."); // Add the possible configurations for this machine @@ -1015,21 +1027,21 @@ //start the monitoring service - fStatVar.sizeWritten = 0; - fStatVar.freeSpace = 0; - fStatVar.writingRate = 0; - fPreviousStatsUpdateTime = Time().Mjd(); +// fStatVar.sizeWritten = 0; +// fStatVar.freeSpace = 0; +// fStatVar.writingRate = 0; +// fPreviousStatsUpdateTime = Time().Mjd(); fPreviousOldRunNumberCheck = Time().Mjd(); - fPreviousSize = 0; - - struct statvfs vfs; - if (!statvfs(fNightlyFilePath.c_str(), &vfs)) - fStatVar.freeSpace = vfs.f_bsize*vfs.f_bavail; - else - fStatVar.freeSpace = -1; - - fStatsMonitoring = new DimDescribedService(GetName() + "/STATS", "X:3", fStatVar, "Add description here"); - - fBaseSizeNightly = 0; - fBaseSizeRun = 0; +// fPreviousSize = 0; + +// struct statvfs vfs; +// if (!statvfs(fNightlyFilePath.c_str(), &vfs)) +// fStatVar.freeSpace = vfs.f_bsize*vfs.f_bavail; +// else +// fStatVar.freeSpace = -1; + +// fStatsMonitoring = new DimDescribedService(GetName() + "/STATS", "X:3", fStatVar, "Add description here"); + +// fBaseSizeNightly = 0; +// fBaseSizeRun = 0; fDailyFileDayChangedAlready = true; fRunNumberTimeout = 60; //default run-timeout set to 1 minute @@ -1066,5 +1078,5 @@ delete fOpenedRunFiles; delete fNumSubAndFits; - delete fStatsMonitoring; +// delete fStatsMonitoring; if (fDebugIsOn) @@ -1077,5 +1089,5 @@ //! checks if the statistic service should be updated, and if so, do it // -void DataLogger::UpdateStatisticsService() +/*void DataLogger::UpdateStatisticsService() { //update the fits files sizes @@ -1104,5 +1116,5 @@ Debug(str); } -} +}*/ // -------------------------------------------------------------------------- // @@ -1192,5 +1204,5 @@ // dim_unlock(); //update the fits files sizes - UpdateStatisticsService(); + // UpdateStatisticsService(); //remove old run numbers @@ -1252,8 +1264,9 @@ //get the size of the newly opened file. #ifdef RUN_LOGS - RememberFileOrigSizePlease(run.logName, false); + fFilesStats.FileOpened(run.logName); +// RememberFileOrigSizePlease(run.logName, false); #endif - RememberFileOrigSizePlease(run.reportName, false); - +// RememberFileOrigSizePlease(run.reportName, false); + fFilesStats.FileOpened(run.reportName); //TODO this notification scheme might be messed up now.... fix it ! const string baseFileName = CompileFileNameWithPath(fRunFilePath, run.runNumber, "", ""); @@ -1602,7 +1615,13 @@ } - DataLoggerStats statVar; - /*const bool statWarning =*/ calculateTotalSizeWritten(statVar, true); - + FileStatisticsData statVar; + fFilesStats.GetTotalSizeWritten(statVar); + // /*const bool statWarning =*/ calculateTotalSizeWritten(statVar, true); +#ifdef HAVE_FITS + str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:"; + Message(str); +#else + Message("FITS output disabled at compilation"); +#endif Message("----------------- STATS ------------------"); str.str(""); @@ -1930,10 +1949,12 @@ } //get the size of the newly opened file. - fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName); - fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName); - fFileSizesMap.clear(); - fBaseSizeRun = 0; - fPreviousSize = 0; - +// fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName); +// fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName); +// fFileSizesMap.clear(); +// fBaseSizeRun = 0; +// fPreviousSize = 0; + fFilesStats.Reset(); + fFilesStats.FileOpened(fFullNightlyLogFileName); + fFilesStats.FileOpened(fFullNightlyReportFileName); //notify that a new file has been opened. const string baseFileName = CompileFileNameWithPath(fNightlyFilePath, "", ""); @@ -1992,5 +2013,6 @@ AllocateFITSBuffers(sub); //get the size of the file we're about to open - if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update) + if (fFilesStats.FileOpened(partialName)) +// if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update) fOpenedNightlyFits[fileNameOnly].push_back(serviceName); @@ -2027,5 +2049,6 @@ } //get the size of the file we're about to open - if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update) + if (fFilesStats.FileOpened(partialName)) +// if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update) cRunNumber->openedFits[fileNameOnly].push_back(serviceName); else @@ -2455,14 +2478,35 @@ // //! Implements the transition towards kSM_WaitingRun +//! If current state is kSM_Ready, then tries to go to nightlyOpen state first. +//! @returns +//! kSM_WaitingRun or kSM_badNightlyConfig +int DataLogger::NightlyToWaitRunPlease() +{ + int cState = GetCurrentState(); + if (cState == kSM_Ready) + cState = StartPlease(); + + if (cState != kSM_NightlyOpen) + return GetCurrentState(); + + if (fDebugIsOn) + { + Debug("Going to Wait Run Number state..."); + } + return kSM_WaitingRun; +} +// -------------------------------------------------------------------------- +// +//! Implements the transition from wait for run number to nightly open //! Does nothing really. //! @returns //! kSM_WaitingRun -int DataLogger::NightlyToWaitRunPlease() +int DataLogger::BackToNightlyOpenPlease() { if (fDebugIsOn) { - Debug("Going to Wait Run Number state..."); - } - return kSM_WaitingRun; + Debug("Going to NightlyOpen state..."); + } + return kSM_NightlyOpen; } // --------------------------------------------------------------------------