Changeset 11463 for trunk/FACT++/src
- Timestamp:
- 07/19/11 14:13:46 (13 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/FACT++/src/datalogger.cc
r11441 r11463 59 59 #include "DimNetwork.h" 60 60 61 61 62 #ifdef HAVE_FITS 62 63 #include "Fits.h" … … 64 65 65 66 //Dim structures 66 ///Distributes the writing statistics67 //struct DataLoggerStats {68 // long sizeWritten;69 // long freeSpace;70 // long writingRate;71 //};72 67 ///distributes the number of opened subscriptions and fits files 73 68 struct NumSubAndFitsType { … … 283 278 void ServicesMonitoring(); 284 279 inline void NotifyOpenedFile(const string &name, int type, DimDescribedService* service); 285 ///variables for computing statistics286 // DataLoggerStats fStatVar;287 ///stores the size of each file that is or was open288 // map<string, long> fFileSizesMap;289 ///total size of the opened files BEFORE they were opened by the logger290 // long fBaseSizeNightly;291 // long fPreviousSize;292 // long fBaseSizeRun;293 280 ///Service for opened files 294 281 DimDescribedService* fOpenedNightlyFiles; 295 282 DimDescribedService* fOpenedRunFiles; 296 283 DimDescribedService* fNumSubAndFits; 297 // DimDescribedService* fStatsMonitoring;298 284 NumSubAndFitsType fNumSubAndFitsData; 299 ///Small function for calculating the total size written so far300 // bool calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting);301 285 302 286 /*************************************************** … … 358 342 ///Open a text file and checks for ofstream status 359 343 bool OpenTextFilePlease(ofstream& stream, const string& name); 360 ///Check if a dir is . and returns the actual string corresponding to .361 // string CheckIfDirIsDot(const string& dir);362 ///Remembers the size of newly opened files. for statistic purposes363 // bool RememberFileOrigSizePlease(string& fileName, bool nightly);364 344 ///Checks if the input osftream is in error state, and if so close it. 365 345 bool CheckForOfstreamError(ofstream& out, bool isDailyStream); … … 369 349 ///Checks if a given path exist 370 350 bool DoesPathExist(string path); 371 ///Check if the statistics service should be updated, and if so, do it372 // void UpdateStatisticsService();373 351 ///Check if old run numbers can be trimmed, and if so, do it 374 352 void TrimOldRunNumbers(); … … 384 362 ///Remove all the services associated with a given server 385 363 void RemoveAllServices(const string&); 386 ///Remove a server and all of its services 387 //void RemoveServer(const std::string &s); 388 ///Remove all the servers and all subscriptions 389 //void RemoveAllServers(); 364 /*************************************************** 365 * Overwritten from MessageImp 366 ***************************************************/ 367 vector<string> backLogBuffer; 368 bool shouldBackLog; 369 public: 370 int Write(const Time &time, const std::string &txt, int qos=kMessage); 371 390 372 }; //DataLogger 391 373 374 // -------------------------------------------------------------------------- 375 // 376 //! Overwritten write function. This way we directly log the datalogger's messages, without going through dim's dns, 377 //! thus increasing robustness. 378 //! @param time: see MessageImp class param 379 //! @param txt: see MessageImp class param 380 //! @param qos: see MessageImp class param 381 //! @return see MessageImp class param 382 // 383 int DataLogger::Write(const Time&time, const std::string& txt, int qos) 384 { 385 if (fNightlyLogFile.is_open()) 386 { 387 MessageImp mimp(fNightlyLogFile); 388 mimp.Write(time, txt, qos); 389 } 390 else if (shouldBackLog) 391 { 392 ostringstream str; 393 MessageImp mimp(str); 394 mimp.Write(time, txt, qos); 395 backLogBuffer.push_back(str.str()); 396 } 397 return StateMachineDim::Write(time, txt, qos); 398 } 392 399 // -------------------------------------------------------------------------- 393 400 // … … 489 496 } 490 497 491 // list[service].dimInfo = shared_ptr<DimStampedInfo>(SubscribeToPlease(server, service));492 498 list[service].dimInfo.reset(SubscribeToPlease(server, service)); 493 499 list[service].server = server; … … 544 550 } 545 551 } 546 /*547 // --------------------------------------------------------------------------548 //549 //! Remove a given server550 //! @param s the server to be removed551 //552 void DataLogger::RemoveServer(const std::string &s)553 {554 if (fServiceSubscriptions.erase(s) != 1)555 {556 Error("The server " + s + " to be removed was not in the servers list");557 }558 DimServiceInfoList::RemoveServer(s);559 }560 // --------------------------------------------------------------------------561 //562 //! Remove all the server and associated subscriptions563 //564 void DataLogger::RemoveAllServers()565 {566 fServiceSubscriptions.clear();567 DimServiceInfoList::RemoveAllServers();568 }*/569 552 570 553 // -------------------------------------------------------------------------- … … 588 571 return false; 589 572 } 590 // --------------------------------------------------------------------------591 //592 //! Checks the size on disk of a given size, and remembers it in the relevant member variable593 //! @param fileName the file for which the size on disk should be retrieved594 //! @param nightly whether this is a run or nightly file, so that its size is added to the correct member variable595 //596 //bool DataLogger::RememberFileOrigSizePlease(string& fileName, bool nightly)597 //{598 //get the size of the file we're about to open599 // if (fFileSizesMap.find(fileName) != fFileSizesMap.end())600 // return false;601 602 // if (nightly)603 // fBaseSizeNightly += GetFileSize(fileName);604 // else605 // fBaseSizeRun += GetFileSize(fileName);606 // fFileSizesMap[fileName] = 0;607 // return true;//608 //}609 573 610 574 // -------------------------------------------------------------------------- … … 617 581 { 618 582 errno = 0; 583 if (!stream.is_open()) 619 584 stream.open(name.c_str(), ios_base::out | ios_base::app); 620 if (!stream )585 if (!stream.is_open()) 621 586 { 622 587 ostringstream str; … … 817 782 // -------------------------------------------------------------------------- 818 783 // 819 //! Calculate the total number of written bytes since the logger was started820 //! @param statVar the data structure that should be updated821 //! @param shouldWarn whether or not error messages should be outputted822 //! @param isPrinting whether this function was called from the PRINT command or not. If so, displays relevant information823 //824 /*825 bool DataLogger::calculateTotalSizeWritten(DataLoggerStats& statVar, bool isPrinting)826 {827 #ifdef HAVE_FITS828 if (isPrinting)829 {830 ostringstream str;831 str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:";832 Message(str);833 }834 835 ///TODO the grouping file is dealt with several times. This should not be a problem but well, better to fix it I guess.836 for (SubscriptionsListType::const_iterator x=fServiceSubscriptions.begin();837 x!=fServiceSubscriptions.end(); x++)838 {839 for (map<string, SubscriptionType>::const_iterator y=x->second.begin();840 y!=x->second.end(); y++)841 {842 if (y->second.runFile.IsOpen())843 {844 fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();845 if (isPrinting)846 Message("-> "+y->second.runFile.fFileName);847 }848 if (y->second.nightlyFile.IsOpen())849 {850 fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();851 if (isPrinting)852 Message("-> "+y->second.nightlyFile.fFileName);853 }854 }855 }856 #else857 if (isPrinting)858 Message("FITS output disabled at compilation");859 #endif860 //gather log and report files sizes on disk861 if (fNightlyLogFile.is_open())862 fFileSizesMap[fFullNightlyLogFileName] = GetFileSize(fFullNightlyLogFileName);863 if (fNightlyReportFile.is_open())864 fFileSizesMap[fFullNightlyReportFileName] = GetFileSize(fFullNightlyReportFileName);865 for (list<RunNumberType>::iterator it = fRunNumber.begin(); it != fRunNumber.end(); it++)866 {867 if (it->reportFile->is_open())868 fFileSizesMap[it->reportName] = GetFileSize(it->reportName);869 #ifdef RUN_LOGS870 if (it->logFile->is_open())871 fFileSizesMap[it->logName] = GetFileSize(it->logName);872 #endif873 }874 875 bool shouldWarn = false;876 struct statvfs vfs;877 if (!statvfs(fNightlyFilePath.c_str(), &vfs))878 statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;879 else880 {881 ostringstream str;882 str << "Unable to retrieve stats for " << fNightlyFilePath << ". Reason: " << strerror(errno) << " [" << errno << "]";883 if (!shouldWarn)884 Error(str);885 statVar.freeSpace = -1;886 }887 //sum up all the file sizes. past and present888 statVar.sizeWritten = 0;889 for (map<string, long>::const_iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++)890 statVar.sizeWritten += it->second;891 statVar.sizeWritten -= fBaseSizeNightly;892 statVar.sizeWritten -= fBaseSizeRun;893 894 return shouldWarn;895 }896 */897 // --------------------------------------------------------------------------898 //899 784 //! Default constructor. The name of the machine is given DATA_LOGGER 900 785 //! and the state is set to kSM_Ready at the end of the function. … … 905 790 fFilesStats("DATA_LOGGER", *this) 906 791 { 792 shouldBackLog = true; 907 793 //initialize member data 908 794 fNightlyFilePath = "."; … … 989 875 990 876 //services parameters 991 fDebugIsOn = false; 992 // fStatsPeriodDuration = 1.0f; 993 fOpenedFilesIsOn = true; 877 fDebugIsOn = false; 878 fOpenedFilesIsOn = true; 994 879 fNumSubAndFitsIsOn = true; 995 880 … … 1022 907 fDestructing = false; 1023 908 1024 //start the monitoring service1025 // fStatVar.sizeWritten = 0;1026 // fStatVar.freeSpace = 0;1027 // fStatVar.writingRate = 0;1028 // fPreviousStatsUpdateTime = Time().Mjd();1029 909 fPreviousOldRunNumberCheck = Time().Mjd(); 1030 // fPreviousSize = 0; 1031 1032 // struct statvfs vfs; 1033 // if (!statvfs(fNightlyFilePath.c_str(), &vfs)) 1034 // fStatVar.freeSpace = vfs.f_bsize*vfs.f_bavail; 1035 // else 1036 // fStatVar.freeSpace = -1; 1037 1038 // fStatsMonitoring = new DimDescribedService(GetName() + "/STATS", "X:3", fStatVar, "Add description here"); 1039 1040 // fBaseSizeNightly = 0; 1041 // fBaseSizeRun = 0; 910 1042 911 fDailyFileDayChangedAlready = true; 1043 912 fRunNumberTimeout = 60; //default run-timeout set to 1 minute … … 1057 926 Debug("DataLogger destruction starts"); 1058 927 1059 //this boolean should not be required anymore , because it928 //this boolean should not be required anymore 1060 929 fDestructing = true; 1061 930 1062 931 //first, let's backup the datalogger/message service subscription 1063 shared_ptr<DimStampedInfo> messageBackup;1064 const SubscriptionsListType::iterator x = fServiceSubscriptions.find("DATA_LOGGER");1065 if (x != fServiceSubscriptions.end())1066 {1067 const map<string, SubscriptionType>::iterator y = x->second.find("MESSAGE");1068 if (y != x->second.end())1069 messageBackup = y->second.dimInfo;1070 }1071 1072 dim_lock();932 // shared_ptr<DimStampedInfo> messageBackup; 933 // const SubscriptionsListType::iterator x = fServiceSubscriptions.find("DATA_LOGGER"); 934 // if (x != fServiceSubscriptions.end()) 935 // { 936 // const map<string, SubscriptionType>::iterator y = x->second.find("MESSAGE"); 937 // if (y != x->second.end()) 938 // messageBackup = y->second.dimInfo; 939 // } 940 941 // dim_lock(); 1073 942 1074 943 //now clear the services subscriptions … … 1083 952 GoToReadyPlease(); 1084 953 1085 dim_unlock();954 // dim_unlock(); 1086 955 1087 956 Info("Will soon close the daily log file"); … … 1092 961 1093 962 //release message service before closing nightly log file 1094 if (messageBackup)1095 messageBackup.reset();963 // if (messageBackup) 964 // messageBackup.reset(); 1096 965 1097 966 if (fNightlyLogFile.is_open())//this file is the only one that has not been closed by GoToReadyPlease 1098 967 { 1099 dim_lock();968 // dim_lock(); 1100 969 fNightlyLogFile.close(); 1101 dim_unlock();970 // dim_unlock(); 1102 971 } 1103 972 … … 1105 974 Debug("DataLogger desctruction ends"); 1106 975 } 1107 // -------------------------------------------------------------------------- 1108 // 1109 //! checks if the statistic service should be updated, and if so, do it 1110 // 1111 /*void DataLogger::UpdateStatisticsService() 1112 { 1113 //update the fits files sizes 1114 const Time cTime = Time(); 1115 1116 if ((fStatsPeriodDuration == 0) || ((cTime - fPreviousStatsUpdateTime).total_seconds() < fStatsPeriodDuration)) 1117 return; 1118 1119 calculateTotalSizeWritten(fStatVar, false); 1120 fStatVar.writingRate = (fStatVar.sizeWritten - fPreviousSize)/((cTime - fPreviousStatsUpdateTime).total_seconds()); 1121 fPreviousSize = fStatVar.sizeWritten; 1122 fPreviousStatsUpdateTime = cTime; 1123 //update the service. No need to check if data has been written, because some must have been, otherwise we would not have hit this piece of code 1124 if (fStatVar.writingRate < 0) 1125 fStatVar.writingRate = 0; 1126 if (fStatVar.sizeWritten < 0) 1127 fStatVar.sizeWritten = 0; 1128 fStatsMonitoring->updateService(); 1129 1130 if(fDebugIsOn) 1131 { 1132 ostringstream str; 1133 str << "Size written: " << fStatVar.sizeWritten/1000 << " kB; writing rate: "; 1134 str << fStatVar.writingRate/1000 << " kB/s; free space: "; 1135 str << fStatVar.freeSpace/(1000*1000) << " MB"; 1136 Debug(str); 1137 } 1138 }*/ 976 1139 977 // -------------------------------------------------------------------------- 1140 978 // … … 1164 1002 if (I==NULL) 1165 1003 return; 1166 1167 //it may happen that we try to write a dimInfo while closing files. Prevent that with a dim_lock()1168 // dim_lock();1169 1004 1170 1005 //check if the service pointer corresponds to something that we subscribed to … … 1187 1022 { 1188 1023 DimServiceInfoList::infoHandler(); 1189 // dim_unlock();1190 1024 return; 1191 1025 } 1192 1026 if (I->getSize() <= 0 || I->getData()==NULL) 1193 1027 { 1194 // dim_unlock();1195 1028 return; 1196 1029 } … … 1207 1040 if (I->getTimestamp() == 0) 1208 1041 { 1209 // dim_unlock();1210 1042 return; 1211 1043 } … … 1217 1049 1218 1050 ReportPlease(I, y->second); 1219 1220 // dim_unlock();1221 //update the fits files sizes1222 // UpdateStatisticsService();1223 1051 1224 1052 //remove old run numbers … … 1281 1109 #ifdef RUN_LOGS 1282 1110 fFilesStats.FileOpened(run.logName); 1283 // RememberFileOrigSizePlease(run.logName, false);1284 1111 #endif 1285 // RememberFileOrigSizePlease(run.reportName, false);1286 1112 fFilesStats.FileOpened(run.reportName); 1287 1113 //TODO this notification scheme might be messed up now.... fix it ! … … 1746 1572 int DataLogger::SetStatsPeriod(const Event& evt) 1747 1573 { 1748 /* const float backupDuration = fStatsPeriodDuration;1749 1750 fStatsPeriodDuration = evt.GetFloat();1751 1752 if (fStatsPeriodDuration < 0)1753 {1754 Error("Statistics period duration should be greater than zero. Discarding provided value.");1755 fStatsPeriodDuration = backupDuration;1756 return GetCurrentState();1757 }1758 if (!finite(fStatsPeriodDuration))// != fStatsPeriodDuration)1759 {1760 Error("Provided duration does not appear to be a valid float. Discarding it.");1761 fStatsPeriodDuration = backupDuration;1762 return GetCurrentState();1763 }1764 if (backupDuration == fStatsPeriodDuration)1765 Warn("Statistics period not modified. Supplied value already in use.");1766 1767 if (fStatsPeriodDuration == 0.0f)1768 Message("Statistics are now OFF");1769 else1770 {1771 ostringstream str;1772 str << "Statistics period is now " << fStatsPeriodDuration << " seconds";1773 Message(str);1774 }1775 */1776 1574 fFilesStats.SetUpdateInterval(evt.GetShort()); 1777 1575 return GetCurrentState(); … … 1987 1785 return kSM_BadNightlyConfig; 1988 1786 } 1989 //get the size of the newly opened file. 1990 // fBaseSizeNightly = GetFileSize(fFullNightlyLogFileName); 1991 // fBaseSizeNightly += GetFileSize(fFullNightlyReportFileName); 1992 // fFileSizesMap.clear(); 1993 // fBaseSizeRun = 0; 1994 // fPreviousSize = 0; 1995 // fFilesStats.Reset(); 1787 1996 1788 fFilesStats.FileOpened(fFullNightlyLogFileName); 1997 1789 fFilesStats.FileOpened(fFullNightlyReportFileName); … … 2053 1845 //get the size of the file we're about to open 2054 1846 if (fFilesStats.FileOpened(partialName)) 2055 // if (RememberFileOrigSizePlease(partialName, true))//and remember that the file was opened (i.e. not an update)2056 1847 fOpenedNightlyFits[fileNameOnly].push_back(serviceName); 2057 1848 … … 2059 1850 { 2060 1851 GoToRunWriteErrorState(); 2061 //SetCurrentState(kSM_WriteError);2062 1852 return; 2063 1853 } … … 2090 1880 //get the size of the file we're about to open 2091 1881 if (fFilesStats.FileOpened(partialName)) 2092 // if (RememberFileOrigSizePlease(partialName, false))//and remember that the file was opened (i.e. not an update)2093 1882 cRunNumber->openedFits[fileNameOnly].push_back(serviceName); 2094 1883 else … … 2109 1898 str << "Open FITS file " << partialName << ": " << e.message(); 2110 1899 Error(str); 2111 cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>(); //NULL;1900 cRunNumber->runFitsFile = shared_ptr<CCfits::FITS>(); 2112 1901 GoToRunWriteErrorState(); 2113 //SetCurrentState(kSM_WriteError);2114 1902 return; 2115 1903 } 2116 1904 2117 1905 const string baseFileName = CompileFileNameWithPath(fRunFilePath, sub.runNumber, "", ""); 2118 NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles); // + '_' + serviceName, 4);1906 NotifyOpenedFile(baseFileName, 7, fOpenedRunFiles); 2119 1907 2120 1908 if (hasGrouping) … … 2123 1911 { 2124 1912 GoToRunWriteErrorState(); 2125 //SetCurrentState(kSM_WriteError);2126 1913 return; 2127 1914 } … … 2132 1919 { 2133 1920 GoToRunWriteErrorState(); 2134 //SetCurrentState(kSM_WriteError);2135 1921 return; 2136 1922 } … … 2216 2002 GoToNightlyWriteErrorState(); 2217 2003 return; 2218 //SetCurrentState(kSM_WriteError);2219 2004 } 2220 2005 } … … 2227 2012 GoToRunWriteErrorState(); 2228 2013 return; 2229 //SetCurrentState(kSM_WriteError);2230 2014 } 2231 2015 } … … 2481 2265 StopRunPlease(); 2482 2266 //it may be that dim tries to write a dimInfo while we're closing files. Prevent that 2483 // dim_lock();2484 2267 const string baseFileName = CompileFileNameWithPath(fNightlyFilePath, "", ""); 2485 //we should log everything that we can, even if in Ready state. thus removing this closing (done only in the destructor or at day-change) 2486 // if (fNightlyLogFile.is_open() && !fDestructing)//in case we're destructing the datalogger, wait before closing the log file. 2487 // { 2488 // fNightlyLogFile.close(); 2489 // Info("Closed: "+baseFileName+".log"); 2490 // } 2268 2491 2269 if (fNightlyReportFile.is_open()) 2492 2270 { … … 2513 2291 CreateFitsGrouping(fOpenedNightlyFits, 0); 2514 2292 #endif 2515 // dim_unlock();2516 2293 return kSM_Ready; 2517 2294 } … … 2567 2344 2568 2345 //Adding entries that should ALWAYS be ignored 2569 //fBlackList.insert("DATA_LOGGER/");2346 fBlackList.insert("DATA_LOGGER/MESSAGE"); 2570 2347 fBlackList.insert("/SERVICE_LIST"); 2571 2348 fBlackList.insert("DIS_DNS/"); … … 2604 2381 if (!OpenTextFilePlease(fNightlyLogFile, fFullNightlyLogFileName)) 2605 2382 SetCurrentState(kSM_BadNightlyConfig); 2606 } 2383 else 2384 { 2385 NotifyOpenedFile(fFullNightlyLogFileName, 1, fOpenedNightlyFiles); 2386 for (vector<string>::iterator it=backLogBuffer.begin();it!=backLogBuffer.end();it++) 2387 fNightlyLogFile << *it << endl; 2388 } 2389 } 2390 2391 shouldBackLog = false; 2392 backLogBuffer.clear(); 2607 2393 2608 2394 //configure the interval between statistics updates … … 2619 2405 } 2620 2406 2621 // --------------------------------------------------------------------------2622 /*2623 int RunDim(Configuration &conf)2624 {2625 WindowLog wout;2626 2627 ReadlineColor::PrintBootMsg(wout, conf.GetName(), false);2628 2629 //log.SetWindow(stdscr);2630 if (conf.Has("log"))2631 if (!wout.OpenLogFile(conf.Get<string>("log")))2632 wout << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;2633 2634 // Start io_service.Run to use the StateMachineImp::Run() loop2635 // Start io_service.run to only use the commandHandler command detaching2636 DataLogger logger(wout);2637 if (!logger.EvalConfiguration(conf))2638 return -1;2639 2640 logger.Run(true);2641 2642 return 0;2643 }2644 */2645 // --------------------------------------------------------------------------2646 2407 2647 2408 #include "Main.h" 2648 /* 2649 void RunThread(DataLogger* logger) 2650 { 2651 // This is necessary so that the StateMachine Thread can signal the 2652 // Readline thread to exit 2653 logger->Run(true); 2654 Readline::Stop(); 2655 } 2656 */ 2409 2657 2410 // -------------------------------------------------------------------------- 2658 2411 template<class T> … … 2660 2413 { 2661 2414 return Main<T, DataLogger>(conf, true); 2662 /*2663 static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);2664 2665 WindowLog &win = shell.GetStreamIn();2666 WindowLog &wout = shell.GetStreamOut();2667 2668 if (conf.Has("log"))2669 if (!wout.OpenLogFile(conf.Get<string>("log")))2670 win << kRed << "ERROR - Couldn't open log-file " << conf.Get<string>("log") << ": " << strerror(errno) << endl;2671 2672 DataLogger logger(wout);2673 2674 if (!logger.EvalConfiguration(conf))2675 return -1;2676 2677 shell.SetReceiver(logger);2678 2679 boost::thread t(boost::bind(RunThread, &logger));2680 2681 if (conf.Has("cmd"))2682 {2683 const vector<string> v = conf.Get<vector<string>>("cmd");2684 for (vector<string>::const_iterator it=v.begin(); it!=v.end(); it++)2685 shell.ProcessLine(*it);2686 }2687 2688 if (conf.Has("exec"))2689 {2690 const vector<string> v = conf.Get<vector<string>>("exec");2691 for (vector<string>::const_iterator it=v.begin(); it!=v.end(); it++)2692 shell.Execute(*it);2693 }2694 2695 if (conf.Get<bool>("quit"))2696 shell.Stop();2697 2698 2699 shell.Run(); // Run the shell2700 2701 logger.Stop();2702 2703 //Wait until the StateMachine has finished its thread2704 //before returning and destroyinng the dim objects which might2705 //still be in use.2706 t.join();2707 2708 return 0;2709 */2710 2415 } 2711 2416
Note:
See TracChangeset
for help on using the changeset viewer.