Index: /trunk/FACT++/src/Fits.cc =================================================================== --- /trunk/FACT++/src/Fits.cc (revision 10528) +++ /trunk/FACT++/src/Fits.cc (revision 10529) @@ -143,18 +143,31 @@ std::string factTableName = "FACT-" + tableName; fTable = fFile->addTable(factTableName, 0, allNames, allDataTypes, allUnits); + fTable->makeThisCurrent(); fCopyBuffer = new unsigned char[fTotalNumBytes]; fNumRows = fTable->rows(); if (fNumRows !=0) {//If the file already existed, then we must load its data to memory before writing to it. - std::vector tableNameString; - tableNameString.push_back(tableName); - fFile->read(tableNameString); + BinTable* bTable = dynamic_cast(fTable); + if (!bTable) + { + fMess->Error("The table " + factTableName + " could not be converted to a binary table. skipping"); + return; + } + //read the table binary data. + std::vector colName; + bTable->readData(true, colName); + + //double check that the data was indeed read from the disk. Go through the fTable instead as colName is empty (yes, it is !) std::map cMap = fTable->column(); std::map::iterator cMapIt; + for (cMapIt = cMap.begin(); cMapIt != cMap.end(); cMapIt++) { - //TODO this only works for scalar columns I assume. upgrade it to fully read vector columns - cMapIt->second->readData(1, fNumRows); - } + if (!cMapIt->second->isRead()) + { + fMess->Error("Column " + cMapIt->first + "Could not be read back from the disk"); + return; + } + } updating = true; } @@ -255,7 +268,7 @@ { - fTable->makeThisCurrent(); try { + fTable->makeThisCurrent(); fTable->insertRows(fNumRows); } Index: /trunk/FACT++/src/dataLogger.cc =================================================================== --- /trunk/FACT++/src/dataLogger.cc (revision 10528) +++ /trunk/FACT++/src/dataLogger.cc (revision 10529) @@ -14,8 +14,8 @@ e [label="Error" color="red"]; r [label="Ready"] - d [label="DailyOpen"] + d [label="NightlyOpen"] w [label="WaitingRun"] l [label="Logging"] - b [label="BadDailyconfig" color="red"] + b [label="BadNightlyconfig" color="red"] c [label="BadRunConfig" color="red"] @@ -55,4 +55,6 @@ #include "Description.h" +#include "DimServiceInfoList.h" + //for getting stat of opened files #include @@ -76,6 +78,6 @@ //Dim structures struct DataLoggerStats { - long long sizeWritten; - long long freeSpace; + long sizeWritten; + long freeSpace; long writingRate; }; @@ -84,4 +86,9 @@ int numSubscriptions; int numOpenFits; +}; + +struct OpenFileToDim { + int code; + char fileName[FILENAME_MAX]; }; //For debugging DIM's services @@ -96,5 +103,5 @@ void setQuality(int){}; }; -class DataLogger : public StateMachineDim, DimInfoHandler +class DataLogger : public StateMachineDim, DimInfoHandler//, DimServiceInfoList //,DimInfoHandler { public: @@ -102,27 +109,27 @@ enum { - kSM_DailyOpen = 20, ///< Daily file openned and writing + kSM_NightlyOpen = 20, ///< Nightly file openned and writing kSM_WaitingRun = 30, ///< waiting for the run number to open the run file kSM_Logging = 40, ///< both files openned and writing - kSM_BadDailyConfig = 0x101, ///< the folder specified for daily logging does not exist or has bad permissions + kSM_BadNightlyConfig = 0x101, ///< the folder specified for Nightly logging does not exist or has bad permissions kSM_BadRunConfig = 0x102, ///< the folder specified for the run logging does not exist or has wrong permissions or no run number } localstates_t; - DataLogger(std::ostream &out); + DataLogger(std::ostream &out); ~DataLogger(); private: //Define all the data structure specific to the DataLogger here - /// ofstream for the dailyLogfile - std::ofstream fDailyLogFile; + /// ofstream for the NightlyLogfile + std::ofstream fNightlyLogFile; /// ofstream for the run-specific Log file std::ofstream fRunLogFile; - /// ofstream for the daily report file - std::ofstream fDailyReportFile; + /// ofstream for the Nightly report file + std::ofstream fNightlyReportFile; /// ofstream for the run-specific report file std::ofstream fRunReportFile; - /// base path of the dailyfile - std::string fDailyFileName; + /// base path of the Nightlyfile + std::string fNightlyFileName; ///base path of the run file std::string fRunFileName; @@ -146,5 +153,9 @@ static const char* fTransWait; static const char* fRunNumberInfo; ///< This is the name of the dimInfo received to specify the run number. It must be updated once the final name will be defined - + static const char* fPrintCommand; + static const char* fDebugOnOff; + static const char* fStatsPeriod; + static const char* fStartStopOpenedFiles; + static const char* fStartStopNumSubsAndFits; //overloading of DIM's infoHandler function void infoHandler(); @@ -153,16 +164,10 @@ ServiceList fServiceList; - - ///A std pair to store both the DimInfo name and the actual DimInfo pointer -// typedef std::pair subscriptionType; - ///All the services to which we've subscribed to. Sorted by server name -// std::map > fServiceSubscriptions; - ///A std pair to store both the DimInfo pointer and the corresponding outputted fits file struct SubscriptionType { #ifdef HAS_FITS - ///daily FITS output file - Fits dailyFile; + ///Nightly FITS output file + Fits nightlyFile; ///run-specific FITS output file Fits runFile; @@ -178,5 +183,5 @@ { #ifdef HAS_FITS - dailyFile = other.dailyFile; + nightlyFile = other.nightlyFile; runFile = other.runFile; #endif @@ -189,5 +194,5 @@ { #ifdef HAS_FITS - dailyFile = other.dailyFile; + nightlyFile = other.nightlyFile; runFile = other.runFile; #endif @@ -219,6 +224,6 @@ delete dimInfo; #ifdef HAS_FITS - if (dailyFile.IsOpen()) - dailyFile.Close(); + if (nightlyFile.IsOpen()) + nightlyFile.Close(); if (runFile.IsOpen()) runFile.Close(); @@ -237,10 +242,9 @@ SubscriptionsListType fServiceSubscriptions; - ///Reporting method for the services info received void ReportPlease(DimInfo* I, SubscriptionType& sub); - ///Configuration of the daily file path - int ConfigureDailyFileName(const Event& evt); + ///Configuration of the nightly file path + int ConfigureNightlyFileName(const Event& evt); ///Configuration fo the file name int ConfigureRunFileName(const Event& evt); @@ -249,4 +253,6 @@ ///logging method for the messages int LogMessagePlease(const Event& evt); + ///print the current state of the dataLogger + int PrintStatePlease(const Event& evt); ///checks whether or not the current info being treated is a run number void CheckForRunNumber(DimInfo* I); @@ -259,6 +265,6 @@ ///stop and reset transition int GoToReadyPlease(); - ///from dailyOpen to waiting transition - int DailyToWaitRunPlease(); + ///from NightlyOpen to waiting transition + int NightlyToWaitRunPlease(); #ifdef HAS_FITS ///Open fits files @@ -286,21 +292,36 @@ bool fContinueMonitoring; ///required for accurate monitoring - std::map fFileSizesMap; - std::string fFullDailyLogFileName; - std::string fFullDailyReportFileName; + std::map fFileSizesMap; + std::string fFullNightlyLogFileName; + std::string fFullNightlyReportFileName; std::string fFullRunLogFileName; std::string fFullRunReportFileName; - long long fBaseSizeDaily; - long long fPreviousSize; - long long fBaseSizeRun; + long fBaseSizeNightly; + long fPreviousSize; + long fBaseSizeRun; ///Service for opened files - DimDescribedService* fOpenedDailyFiles; + DimDescribedService* fOpenedNightlyFiles; DimDescribedService* fOpenedRunFiles; DimDescribedService* fNumSubAndFits; NumSubAndFitsType fNumSubAndFitsData; -// char* fDimBuffer; - inline void NotifyOpenedFile(std::string name, int type, DimService* service); - inline void NotifyOpenedFile(std::string , int , MyService* ){} - + + inline void NotifyOpenedFile(std::string name, int type, DimDescribedService* service); +public: + void setBlackWhiteList(const std::string& , bool); +private: + std::set fGreyList; + bool fIsBlackList; + bool fDebugIsOn; + float fStatsPeriodDuration; + bool fOpenedFilesIsOn; + bool fNumSubAndFitsIsOn; + //functions for controlling the services behavior + int SetDebugOnOff(const Event& evt); + int SetStatsPeriod(const Event& evt); + int SetOpenedFilesOnOff(const Event& evt); + int SetNumSubsAndFitsOnOff(const Event& evt); + bool getBooleanFromString(const std::string&); + ///boolean to prevent DIM update while desctructing the dataLogger + bool fDestructing; }; //DataLogger @@ -318,10 +339,14 @@ const char* DataLogger::fTransWait = "WAIT_RUN_NUMBER"; const char* DataLogger::fRunNumberInfo = "RUN_NUMBER"; - +const char* DataLogger::fPrintCommand = "PRINT"; +const char* DataLogger::fDebugOnOff = "DEBUG"; +const char* DataLogger::fStatsPeriod = "STATS_PERIOD"; +const char* DataLogger::fStartStopOpenedFiles = "OPENED_FILES_SRVC"; +const char* DataLogger::fStartStopNumSubsAndFits = "NUM_SUBS_SRVC"; void DataLogger::ServicesMonitoring() { //create the DIM service - int dataSize = 2*sizeof(long long) + sizeof(long); + int dataSize = 2*sizeof(long) + sizeof(long); DataLoggerStats statVar; @@ -331,12 +356,10 @@ struct statvfs vfs; - if (!statvfs(fDailyFileName.c_str(), &vfs)) + if (!statvfs(fNightlyFileName.c_str(), &vfs)) statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; else statVar.freeSpace = -1; -// DimDescribedService srvc((GetName()+"/STATS").c_str(), "x:2;l:1", &statVar, dataSize,//static_cast(data), dataSize, -// "Add description here"); - DimDescribedService srvc ("DATA_LOGGER/STATS", "X:2;L:1", &statVar, dataSize, "Add description here"); - double deltaT = 1; + + DimDescribedService srvc ("DATA_LOGGER/STATS", "X:3", &statVar, dataSize, "Add description here"); fPreviousSize = 0; bool statWarning = false; @@ -344,5 +367,11 @@ while (fContinueMonitoring) { - sleep(deltaT); + if (fStatsPeriodDuration == 0.0f) + { + sleep(0.1f); + continue; + } + else + sleep(fStatsPeriodDuration); //update the fits files sizes #ifdef HAS_FITS @@ -357,8 +386,10 @@ { fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); +#ifdef ONE_FITS_ONLY runFileDone = true; +#endif } - if (y->second.dailyFile.IsOpen()) - fFileSizesMap[y->second.dailyFile.fFileName] = y->second.dailyFile.GetWrittenSize(); + if (y->second.nightlyFile.IsOpen()) + fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); } } @@ -366,13 +397,13 @@ struct stat st; //gather log and report files sizes on disk - if (fDailyLogFile.is_open()) - { - stat(fFullDailyLogFileName.c_str(), &st); - fFileSizesMap[fFullDailyLogFileName] = st.st_size; - } - if (fDailyReportFile.is_open()) - { - stat(fFullDailyReportFileName.c_str(), &st); - fFileSizesMap[fFullDailyReportFileName] = st.st_size; + if (fNightlyLogFile.is_open()) + { + stat(fFullNightlyLogFileName.c_str(), &st); + fFileSizesMap[fFullNightlyLogFileName] = st.st_size; + } + if (fNightlyReportFile.is_open()) + { + stat(fFullNightlyReportFileName.c_str(), &st); + fFileSizesMap[fFullNightlyReportFileName] = st.st_size; } if (fRunLogFile.is_open()) @@ -387,5 +418,5 @@ } - if (!statvfs(fDailyFileName.c_str(), &vfs)) + if (!statvfs(fNightlyFileName.c_str(), &vfs)) { statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; @@ -395,5 +426,5 @@ { std::stringstream str; - str << "Unable to retrieve stats for " << fDailyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; + str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; if (!statWarning) Error(str); @@ -404,15 +435,24 @@ //sum up all the file sizes. past and present statVar.sizeWritten = 0; - for (std::map::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) + for (std::map::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) statVar.sizeWritten += it->second; - statVar.sizeWritten -= fBaseSizeDaily; + statVar.sizeWritten -= fBaseSizeNightly; statVar.sizeWritten -= fBaseSizeRun; - //FIXME get the actual time elapsed - statVar.writingRate = (statVar.sizeWritten - fPreviousSize)/deltaT; + if (fStatsPeriodDuration == 0.0f) + continue; + statVar.writingRate = (statVar.sizeWritten - fPreviousSize)/fStatsPeriodDuration; + fPreviousSize = statVar.sizeWritten; if (statVar.writingRate != 0) //if data has been written { -//std::cout << "rate: " << statVar.writingRate << std::endl; - srvc.updateService(&statVar, dataSize);//static_cast(data), dataSize); + srvc.updateService(); + if(fDebugIsOn) + { + stringstream str; + str << "Size written: " << statVar.sizeWritten/1024 << " KB; writting rate: "; + str << statVar.writingRate/1024 << " KB/s; free space: "; + str << statVar.freeSpace/(1024*1024) << " MB"; + Debug(str.str()); + } } } @@ -430,5 +470,5 @@ { //initialize member data - fDailyFileName = ".";//"/home/lyard/log";// + fNightlyFileName = ".";//"/home/lyard/log";// fRunFileName = ".";//"/home/lyard/log"; fRunNumber = 12345; @@ -440,52 +480,57 @@ //Give a name to this machine's specific states - AddStateName(kSM_DailyOpen, "DailyFileOpen", "Add description here"); + AddStateName(kSM_NightlyOpen, "NightlyFileOpen", "Add description here"); AddStateName(kSM_WaitingRun, "WaitForRun", "Add description here"); AddStateName(kSM_Logging, "Logging", "Add description here"); - AddStateName(kSM_BadDailyConfig, "ErrDailyFolder", "Add description here"); + AddStateName(kSM_BadNightlyConfig, "ErrNightlyFolder", "Add description here"); AddStateName(kSM_BadRunConfig, "ErrRunFolder", "Add description here"); /*Add the possible transitions for this machine*/ - AddTransition(kSM_DailyOpen, fTransStart, kSM_Ready, kSM_BadDailyConfig) - (boost::bind(&DataLogger::StartPlease, this)); -// ("start the daily logging. daily file location must be specified already"); - - AddTransition(kSM_Ready, fTransStop, kSM_DailyOpen, kSM_WaitingRun, kSM_Logging) - (boost::bind(&DataLogger::GoToReadyPlease, this)) + AddTransition(kSM_NightlyOpen, fTransStart, kSM_Ready, kSM_BadNightlyConfig) + (boost::bind(&DataLogger::StartPlease, this)) + ("start the Nightly logging. Nightly file location must be specified already"); + + AddTransition(kSM_Ready, fTransStop, kSM_NightlyOpen, kSM_WaitingRun, kSM_Logging) + (boost::bind(&DataLogger::GoToReadyPlease, this)) ("stop the data logging"); - AddTransition(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig) - (boost::bind(&DataLogger::StartRunPlease, this)) - ("start the run logging. run file location must be specified already."); - - AddTransition(kSM_WaitingRun, fTransStopRun, kSM_Logging) - (boost::bind(&DataLogger::StopRunPlease, this)) - (""); - - AddTransition(kSM_Ready, fTransReset, kSM_Error, kSM_BadDailyConfig, kSM_BadRunConfig, kSM_Error) - (boost::bind(&DataLogger::GoToReadyPlease, this)) - ("transition to exit error states. dunno if required or not, would close the daily file if already openned."); - - AddTransition(kSM_WaitingRun, fTransWait, kSM_DailyOpen) - (boost::bind(&DataLogger::DailyToWaitRunPlease, this)); - - /*Add the possible configurations for this machine*/ - - AddConfiguration(fConfigDay, "C", kSM_Ready, kSM_BadDailyConfig) - (boost::bind(&DataLogger::ConfigureDailyFileName, this, _1)) - ("configure the daily file location. cannot be done before the file is actually opened"); - - AddConfiguration(fConfigRun, "C", kSM_Ready, kSM_BadDailyConfig, kSM_DailyOpen, kSM_WaitingRun, kSM_BadRunConfig) - (boost::bind(&DataLogger::ConfigureRunFileName, this, _1)) - ("configure the run file location. cannot be done before the file is actually opened, and not in a dailly related error."); - + AddTransition(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig) + (boost::bind(&DataLogger::StartRunPlease, this)) + ("start the run logging. run file location must be specified already."); + + AddTransition(kSM_WaitingRun, fTransStopRun, kSM_Logging) + (boost::bind(&DataLogger::StopRunPlease, this)) + (""); + + AddTransition(kSM_Ready, fTransReset, kSM_Error, kSM_BadNightlyConfig, kSM_BadRunConfig, kSM_Error) + (boost::bind(&DataLogger::GoToReadyPlease, this)) + ("transition to exit error states. dunno if required or not, would close the Nightly file if already openned."); + + AddTransition(kSM_WaitingRun, fTransWait, kSM_NightlyOpen) + (boost::bind(&DataLogger::NightlyToWaitRunPlease, this)); + + /*Add the possible configurations for this machine*/ + AddConfiguration(fConfigDay, "C", kSM_Ready, kSM_BadNightlyConfig) + (boost::bind(&DataLogger::ConfigureNightlyFileName, this, _1)) + ("configure the Nightly file location. cannot be done before the file is actually opened"); + + AddConfiguration(fConfigRun, "C", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig) + (boost::bind(&DataLogger::ConfigureRunFileName, this, _1)) + ("configure the run file location. cannot be done before the file is actually opened, and not in a dailly related error."); + AddConfiguration(fConfigRunNumber, "I", kSM_Ready, kSM_BadNightlyConfig, kSM_NightlyOpen, kSM_WaitingRun, kSM_BadRunConfig) + (boost::bind(&DataLogger::ConfigureRunNumber, this, _1)) + ("configure the run number. cannot be done in logging state"); //Provide a logging command //I get the feeling that I should be going through the EventImp - //instead of DimCommand directly, mainly because the commandHandler - //is already done in StateMachineImp.cc - //Thus I'll simply add a configuration, which I will treat as the logging command - AddConfiguration(fConfigLog, "C", kSM_DailyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadRunConfig) - (boost::bind(&DataLogger::LogMessagePlease, this, _1)); - + //instead of DimCommand directly, mainly because the commandHandler + //is already done in StateMachineImp.cc + //Thus I'll simply add a configuration, which I will treat as the logging command + AddConfiguration(fConfigLog, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadRunConfig) + (boost::bind(&DataLogger::LogMessagePlease, this, _1)); + + //Provide a print command + AddConfiguration(fPrintCommand, "", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_BadNightlyConfig, kSM_BadRunConfig) + (boost::bind(&DataLogger::PrintStatePlease, this, _1)); + fServiceList.SetHandler(this); CheckForServicesUpdate(); @@ -494,18 +539,37 @@ fContinueMonitoring = true; fMonitoringThread = boost::thread(boost::bind(&DataLogger::ServicesMonitoring, this)); - fBaseSizeDaily = 0; + fBaseSizeNightly = 0; fBaseSizeRun = 0; - - //gives the entire buffer size. Otherwise, dim overwrites memory at bizarre locations if smaller size is given at creation time. -// fOpenedFiles = new DimDescribedService((GetName()+"/FILENAME").c_str(), "i:1;C", static_cast(fDimBuffer), 256, "Add description here"); - fOpenedDailyFiles = new DimDescribedService((GetName() + "/FILENAME_DAILY").c_str(), const_cast(""), "Add description here");//"i:1;C", static_cast(fDimBuffer), 256); - fOpenedRunFiles = new DimDescribedService((GetName() + "/FILENAME_RUN").c_str(), const_cast(""), "Add description here"); - fOpenedDailyFiles->setQuality(0); - fOpenedRunFiles->setQuality(0); - fOpenedDailyFiles->updateService(); - fOpenedRunFiles->updateService(); + OpenFileToDim fToDim; + fToDim.code = 0; + fToDim.fileName[0] = '\0'; + fOpenedNightlyFiles = new DimDescribedService((GetName() + "/FILENAME_NIGHTLY").c_str(), "I:1;C", &fToDim, sizeof(int)+1, "Add description here");//"i:1;C", static_cast(fDimBuffer), 256); + fOpenedRunFiles = new DimDescribedService((GetName() + "/FILENAME_RUN").c_str(), "I:1;C", &fToDim, sizeof(int)+1, "Add description here"); fNumSubAndFitsData.numSubscriptions = 0; fNumSubAndFitsData.numOpenFits = 0; fNumSubAndFits = new DimDescribedService((GetName() + "/NUM_SUBS").c_str(), "I:2", &fNumSubAndFitsData, sizeof(NumSubAndFitsType), "Add description here"); + //black/white list + fIsBlackList = true; + fGreyList.clear(); + + //services parameters + fDebugIsOn = false; + fStatsPeriodDuration = 1.0f; + fOpenedFilesIsOn = true; + fNumSubAndFitsIsOn = true; + //provide services control commands + AddConfiguration(fDebugOnOff, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready) + (boost::bind(&DataLogger::SetDebugOnOff, this, _1)); + AddConfiguration(fStatsPeriod, "F", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready) + (boost::bind(&DataLogger::SetStatsPeriod, this, _1)); + AddConfiguration(fStartStopOpenedFiles, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready) + (boost::bind(&DataLogger::SetOpenedFilesOnOff ,this, _1)); + AddConfiguration(fStartStopNumSubsAndFits, "C", kSM_NightlyOpen, kSM_Logging, kSM_WaitingRun, kSM_Ready) + (boost::bind(&DataLogger::SetNumSubsAndFitsOnOff, this, _1)); + fDestructing = false; + if(fDebugIsOn) + { + Debug("DataLogger Init Done."); + } } // -------------------------------------------------------------------------- @@ -543,7 +607,10 @@ for (std::vector::const_iterator i=serverList.begin(); i!=serverList.end();i++) { - //skip the two obvious excluded services + //skip the two de-fact excluded services + //Dim crashes if the publisher subscribes to its own service. This sounds weird, I agree. if ((i->find("DIS_DNS") != std::string::npos) || (i->find("DATA_LOGGER") != std::string::npos)) + continue; + if (fIsBlackList && (fGreyList.find(*i) != fGreyList.end())) continue; //find the current server in our subscription list @@ -575,8 +642,24 @@ if (*givenSubs == "SERVICE_LIST") continue; + if (fIsBlackList && fGreyList.find(*givenSubs) != fGreyList.end()) + continue; + + if (fIsBlackList && fGreyList.find((*i) + "/" + (*givenSubs)) != fGreyList.end()) + continue; + else if (!fIsBlackList && + (fGreyList.find((*i) + "/" + (*givenSubs)) == fGreyList.end()) && + (fGreyList.find(*i) == fGreyList.end()) && + (fGreyList.find(*givenSubs) == fGreyList.end())) + continue; if (cSubs->second.find(*givenSubs) == cSubs->second.end()) {//service not found. Add it cSubs->second[*givenSubs].dimInfo = new DimStampedInfo(((*i) + "/" + *givenSubs).c_str(), const_cast(""), this); serviceUpdated = true; + if(fDebugIsOn) + { + stringstream str; + str << "Subscribing to service " << *i << "/" << *givenSubs; + Debug(str.str()); + } } } @@ -590,6 +673,23 @@ if (*j == "SERVICE_LIST") continue; + if (fIsBlackList && fGreyList.find(*j) != fGreyList.end()) + continue; + + if (fIsBlackList && fGreyList.find((*i) + "/" + (*j)) != fGreyList.end()) + continue; + else if (!fIsBlackList && + (fGreyList.find((*i) + "/" + (*j)) == fGreyList.end()) && + (fGreyList.find(*i) == fGreyList.end()) && + (fGreyList.find(*j) == fGreyList.end())) + continue; + liste[*j].dimInfo = new DimStampedInfo(((*i) + "/" + (*j)).c_str(), const_cast(""), this); serviceUpdated = true; + if(fDebugIsOn) + { + stringstream str; + str << "Subscribing to service " << *i << "/" << *j; + Debug(str.str()); + } } } @@ -603,4 +703,12 @@ DataLogger::~DataLogger() { + if (fDebugIsOn) + { + Debug("DataLogger destruction starts"); + } + fDestructing = true; + //first let's go to the ready state + //TODO some closing done below has already been executed by GoToReady. figure out what should be removed. + GoToReadyPlease(); //release the services subscriptions fServiceSubscriptions.clear(); @@ -610,13 +718,13 @@ fMonitoringThread.join(); //close the files - if (fDailyLogFile.is_open()) - fDailyLogFile.close(); - if (fDailyReportFile.is_open()) - fDailyReportFile.close(); + if (fNightlyLogFile.is_open()) + fNightlyLogFile.close(); + if (fNightlyReportFile.is_open()) + fNightlyReportFile.close(); if (fRunLogFile.is_open()) fRunLogFile.close(); if (fRunReportFile.is_open()) fRunReportFile.close(); - delete fOpenedDailyFiles; + delete fOpenedNightlyFiles; delete fOpenedRunFiles; delete fNumSubAndFits; @@ -629,4 +737,8 @@ #endif #endif + if (fDebugIsOn) + { + Debug("DataLogger desctruction ends"); + } } @@ -637,4 +749,6 @@ void DataLogger::infoHandler() { + if (fDestructing) + return; DimInfo* I = getInfo(); SubscriptionsListType::iterator x; @@ -648,5 +762,14 @@ for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++) fNumSubAndFitsData.numSubscriptions += x->second.size(); - fNumSubAndFits->updateService(); + if (fNumSubAndFitsIsOn) + { + if (fDebugIsOn) + { + stringstream str; + str << "Updating number of subscriptions service: Num Subs=" << fNumSubAndFitsData.numSubscriptions << " Num open FITS=" << fNumSubAndFitsData.numOpenFits; + Debug(str.str()); + } + fNumSubAndFits->updateService(); + } } return; @@ -692,4 +815,7 @@ //TODO check the format here fRunNumber = I->getInt(); + stringstream str; + str << "New run number is " << fRunNumber; + Message(str.str()); } } @@ -705,13 +831,20 @@ //should we log or report this info ? (i.e. is it a message ?) bool isItaReport = ((strstr(I->getName(), "Message") == NULL) && (strstr(I->getName(), "MESSAGE") == NULL)); - + if (I->getFormat()[0] == 'C') + isItaReport = false; //TODO add service exclusion - if (!fDailyReportFile.is_open()) + if (!fNightlyReportFile.is_open()) return; //create the converter for that service - if (sub.fConv == NULL) - { + if (sub.fConv == NULL && isItaReport) + { + //trick the converter in case of 'C'. why do I do this ? well simple: the converter checks that the right number + //of bytes was written. because I skip 'C' with fits, the bytes will not be allocated, hence the "size copied ckeck" + //of the converter will fail, hence throwing an exception. + std::string fakeFormat(I->getFormat()); + if (fakeFormat[fakeFormat.size()-1] == 'C') + fakeFormat = fakeFormat.substr(0, fakeFormat.size()-1); sub.fConv = new Converter(Out(), I->getFormat()); if (!sub.fConv) @@ -764,57 +897,78 @@ replace_if(text.begin(), text.end(), std::ptr_fun(&std::iscntrl), ' '); - //write entry to daily report - try - { - if (fDailyReportFile.is_open()) - fDailyReportFile << header.str() << text << std::endl; - } - catch (std::exception e) - { - std::stringstream str; - str << "Error while writing to daily report file: " << e.what(); - Error(str); + //write entry to Nightly report + if (fNightlyReportFile.is_open()) + { + if (fDebugIsOn) + { + stringstream str; + str << "Writing: \"" << header.str() << text << "\" to Nightly report file"; + Debug(str.str()); + } + fNightlyReportFile << header.str() << text << std::endl; + //check if either eof, bailbit or batbit are set + if (!fNightlyReportFile.good()) + { + Error("An error occured while writing to the nightly report file. Closing it"); + if (fNightlyReportFile.is_open()) + fNightlyReportFile.close(); + } } //write entry to run-report - try - { - if (fRunReportFile.is_open()) - fRunReportFile << header.str() << text << std::endl; - } - catch (std::exception e) - { - std::stringstream str; - str << "Error while writing to run report file: " << e.what(); - Error(str); + if (fRunReportFile.is_open()) + { + if (fDebugIsOn) + { + stringstream str; + str << "Writing: \"" << header.str() << text << "\" to Run report file"; + Debug(str.str()); + } + fRunReportFile << header.str() << text << std::endl; + if (!fRunReportFile.good()) + { + Error("An error occured while writing to the run report file. Closing it."); + if (fRunReportFile.is_open()) + fRunReportFile.close(); + } } } else - {//write entry to both daily and run logs + {//write entry to both Nightly and run logs std::string n = I->getName(); std::stringstream msg; msg << n.substr(0, n.find_first_of('/')) << ": " << I->getString(); - try - { - MessageImp dailyMess(fDailyLogFile); - dailyMess.Write(cTime, msg.str().c_str(), fQuality); - } - catch (std::exception e) - { - std::stringstream str; - str << "Error while writing to daily log file: " << e.what(); - Error(str); + + if (fNightlyLogFile.is_open()) + { + if (fDebugIsOn) + { + stringstream str; + str << "Writing: \"" << msg.str() << "\" to Nightly log file"; + Debug(str.str()); + } + MessageImp nightlyMess(fNightlyLogFile); + nightlyMess.Write(cTime, msg.str().c_str(), fQuality); + if (!fNightlyLogFile.good()) + { + Error("An error occured while writing to the nightly log file. Closing it."); + if (fNightlyLogFile.is_open()) + fNightlyLogFile.close(); + } } if (fRunLogFile.is_open()) { - try - { - MessageImp runMess(fRunLogFile); - runMess.Write(cTime, msg.str().c_str(), fQuality); - } - catch (std::exception e) - { - std::stringstream str; - str << "Error while writing to run log file: " << e.what(); - Error(str); + if (fDebugIsOn) + { + stringstream str; + str << "Writing: \"" << msg.str() << "\" to Run log file"; + Debug(str.str()); + } + MessageImp runMess(fRunLogFile); + runMess.Write(cTime, msg.str().c_str(), fQuality); + if (!fRunLogFile.good()) + { + Error("An error occured while writing to the run log file. Closing it."); + if (fRunLogFile.is_open()) + fRunLogFile.close(); } } @@ -822,8 +976,10 @@ #ifdef HAS_FITS - if (!sub.dailyFile.IsOpen() || !sub.runFile.IsOpen()) - OpenFITSFilesPlease(sub); - WriteToFITS(sub); - + if (isItaReport) + { + if (!sub.nightlyFile.IsOpen() || !sub.runFile.IsOpen()) + OpenFITSFilesPlease(sub); + WriteToFITS(sub); + } #endif @@ -845,5 +1001,5 @@ int DataLogger::LogMessagePlease(const Event& evt) { - if (!fDailyLogFile.is_open()) + if (!fNightlyLogFile.is_open()) return GetCurrentState(); @@ -879,20 +1035,333 @@ replace(text.begin(), text.end(), '\n', '\\'); replace_if(text.begin(), text.end(), std::ptr_fun(&std::iscntrl), ' '); - - if (fDailyLogFile.is_open()) - fDailyLogFile << header; + if (fDebugIsOn) + { + stringstream str; + str << "Logging: \"" << header << text << "\""; + Debug(str.str()); + } + + if (fNightlyLogFile.is_open()) + { + fNightlyLogFile << header; + if (!fNightlyLogFile.good()) + { + Error("An error occured while writing to the run log file. Closing it."); + if (fNightlyLogFile.is_open()) + fNightlyLogFile.close(); + } + } if (fRunLogFile.is_open()) + { fRunLogFile << header; - - if (fDailyLogFile.is_open()) - fDailyLogFile << text; + if (!fRunLogFile.good()) + { + Error("An error occured while writing to the run log file. Closing it."); + if (fRunLogFile.is_open()) + fRunLogFile.close(); + } + } + if (fNightlyLogFile.is_open()) + { + fNightlyLogFile << text; + if (!fNightlyLogFile.good()) + { + Error("An error occured while writing to the run log file. Closing it."); + if (fNightlyLogFile.is_open()) + fNightlyLogFile.close(); + } + } if (fRunLogFile.is_open()) + { fRunLogFile << text; - + if (!fRunLogFile.good()) + { + Error("An error occured while writing to the run log file. Closing it."); + if (fRunLogFile.is_open()) + fRunLogFile.close(); + } + } return GetCurrentState(); } // -------------------------------------------------------------------------- // -//! Sets the path to use for the daily log file. +//! print the dataLogger's current state. invoked by the PRINT command +//! @param evt +//! the current event. Not used by the method +//! @returns +//! the new state. Which, in that case, is the current state +//! +int DataLogger::PrintStatePlease(const Event& ) +{ + Message("-----------------------------------------"); + Message("------DATA LOGGER CURRENT STATE----------"); + Message("-----------------------------------------"); + //print the path configuration + std::string actualTargetDir; + if (fNightlyFileName == ".") + { + char currentPath[FILENAME_MAX]; + getcwd(currentPath, sizeof(currentPath)); + actualTargetDir = currentPath; + } + else + actualTargetDir = fNightlyFileName; + Message("Nightly Path: " + actualTargetDir); + if (fRunFileName == ".") + { + char currentPath[FILENAME_MAX]; + getcwd(currentPath, sizeof(currentPath)); + actualTargetDir = currentPath; + } + else + actualTargetDir = fRunFileName; + Message("Run Path: " + actualTargetDir); + stringstream str; + str << "Run Number: " << fRunFileName; + Message(str.str()); + Message("-----------OPENED FILES------------------"); + //print all the open files. + if (fNightlyLogFile.is_open()) + Message("Nightly Log..........OPEN"); + else + Message("Nightly log........CLOSED"); + if (fNightlyReportFile.is_open()) + Message("Nightly Report.......OPEN"); + else + Message("Nightly Report.....CLOSED"); + if (fRunLogFile.is_open()) + Message("Run Log..............OPEN"); + else + Message("Run Log............CLOSED"); + if (fRunReportFile.is_open()) + Message("Run Report...........OPEN"); + else + Message("Run Report.........CLOSED"); +#ifdef HAS_FITS + str.str(""); + str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:"; + Message(str.str()); + SubscriptionsListType::iterator x; + std::map::iterator y; + bool runFileDone = false; + for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++) + { + for (y=x->second.begin(); y != x->second.end(); y++) + { + if (y->second.runFile.IsOpen() && !runFileDone) + { + fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); + str.str(""); + str << ">>>" << y->second.runFile.fFileName; + Message(str.str()); +#ifdef ONE_FITS_ONLY + runFileDone = true; +#endif + } + if (y->second.nightlyFile.IsOpen()) + { + fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); + str.str(""); + str << ">>>" << y->second.nightlyFile.fFileName; + Message(str.str()); + } + } + } +#else + Message("FITS output disabled at compilation"); +#endif + struct stat st; + DataLoggerStats statVar; + //gather log and report files sizes on disk + if (fNightlyLogFile.is_open()) + { + stat(fFullNightlyLogFileName.c_str(), &st); + fFileSizesMap[fFullNightlyLogFileName] = st.st_size; + } + if (fNightlyReportFile.is_open()) + { + stat(fFullNightlyReportFileName.c_str(), &st); + fFileSizesMap[fFullNightlyReportFileName] = st.st_size; + } + if (fRunLogFile.is_open()) + { + stat(fFullRunLogFileName.c_str(), &st); + fFileSizesMap[fFullRunLogFileName] = st.st_size; + } + if (fRunReportFile.is_open()) + { + stat(fFullRunReportFileName.c_str(), &st); + fFileSizesMap[fFullRunReportFileName] = st.st_size; + } + struct statvfs vfs; + if (!statvfs(fNightlyFileName.c_str(), &vfs)) + { + statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; + } + else + { + str.str(""); + str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; + Error(str);; + statVar.freeSpace = -1; + } + + //sum up all the file sizes. past and present + statVar.sizeWritten = 0; + for (std::map::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) + statVar.sizeWritten += it->second; + statVar.sizeWritten -= fBaseSizeNightly; + statVar.sizeWritten -= fBaseSizeRun; + str.str(""); + str << "Total Size written: " << statVar.sizeWritten; + str << " Disk free space: " << statVar.freeSpace; + Message("-----------------STATS-------------------"); + Message(str.str()); + Message("------------DIM SUBSCRIPTIONS------------"); + str.str(""); + str << "There are " << fNumSubAndFitsData.numSubscriptions << " active DIM subscriptions:"; + Message(str.str()); + str.str(""); + for (std::map>::const_iterator it=fServiceSubscriptions.begin(); it!= fServiceSubscriptions.end();it++) + { + str << "Server " << it->first; + Message(str.str()); + for (std::map::const_iterator it2=it->second.begin(); it2!=it->second.end(); it2++) + { + str.str(""); + str << " " << it2->first; + Message(str.str()); + } + } + Message("-----------------------------------------"); + + return GetCurrentState(); +} +// -------------------------------------------------------------------------- +// +//! figure out whether the string means on or off +//! @param evt +//! the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1 +//! @returns +//! the equivalent boolean +//! +bool DataLogger::getBooleanFromString(const std::string& str) +{ + if (str == "On" || str == "ON" || str == "on" || str == "1") + return true; + if (str == "Off" || str == "OFF" || str == "off" || str == "0") + return false; + Error("Argument could not be converted to boolean. please use On/on/ON/1 or Off/off/OFF/0. defaulting to false."); + return false; +} +// -------------------------------------------------------------------------- +// +//! turn debug mode on and off +//! @param evt +//! the current event. contains the instruction string: On, Off, on, off, ON, OFF, 0 or 1 +//! @returns +//! the new state. Which, in that case, is the current state +//! +int DataLogger::SetDebugOnOff(const Event& evt) +{ + bool backupDebug = fDebugIsOn; + fDebugIsOn = getBooleanFromString(evt.GetText()); + if (fDebugIsOn == backupDebug) + Warn("Warning: debug mode was already in the requested state"); + else + { + stringstream str; + str << "Debug mode is now " << evt.GetText(); + Message(str.str()); + } + return GetCurrentState(); +} +// -------------------------------------------------------------------------- +// +//! set the statistics update period duration. 0 disables the statistics +//! @param evt +//! the current event. contains the new duration. +//! @returns +//! the new state. Which, in that case, is the current state +//! +int DataLogger::SetStatsPeriod(const Event& evt) +{ + float backupDuration = fStatsPeriodDuration; + fStatsPeriodDuration = evt.GetFloat(); + if (fStatsPeriodDuration < 0) + { + Error("Statistics period duration should be greater than zero. Discarding provided value."); + fStatsPeriodDuration = backupDuration; + return GetCurrentState(); + } + if (fStatsPeriodDuration != fStatsPeriodDuration) + { + Error("Provided duration does not appear to be a valid float. discarding it."); + fStatsPeriodDuration = backupDuration; + return GetCurrentState(); + } + if (backupDuration == fStatsPeriodDuration) + Warn("Warning: statistics period was not modified: supplied value already in use"); + else + { + if (fStatsPeriodDuration == 0.0f) + Message("Statistics are now OFF"); + else + { + stringstream str; + str << "Statistics period is now " << fStatsPeriodDuration << " seconds"; + Message(str.str()); + } + } + return GetCurrentState(); +} +// -------------------------------------------------------------------------- +// +//! set the opened files service on or off. +//! @param evt +//! the current event. contains the instruction string. similar to setdebugonoff +//! @returns +//! the new state. Which, in that case, is the current state +//! +int DataLogger::SetOpenedFilesOnOff(const Event& evt) +{ + bool backupOpened = fOpenedFilesIsOn; + fOpenedFilesIsOn = getBooleanFromString(evt.GetText()); + if (fOpenedFilesIsOn == backupOpened) + Warn("Warning: opened files service mode was already in the requested state"); + else + { + stringstream str; + str << "Opened files service mode is now " << evt.GetText(); + Message(str.str()); + } + return GetCurrentState(); + +} +// -------------------------------------------------------------------------- +// +//! set the number of subscriptions and opened fits on and off +//! @param evt +//! the current event. contains the instruction string. similar to setdebugonoff +//! @returns +//! the new state. Which, in that case, is the current state +//! +int DataLogger::SetNumSubsAndFitsOnOff(const Event& evt) +{ + bool backupSubs = fNumSubAndFitsIsOn; + fNumSubAndFitsIsOn = getBooleanFromString(evt.GetText()); + if (fNumSubAndFitsIsOn == backupSubs) + Warn("Warning: Number of subscriptions service mode was already in the requested state"); + else + { + stringstream str; + str << "Number of subscriptions service mode is now " << evt.GetText(); + Message(str.str()); + } + return GetCurrentState(); +} +// -------------------------------------------------------------------------- +// +//! Sets the path to use for the Nightly log file. //! @param evt //! the event transporting the path @@ -900,13 +1369,13 @@ //! currently only the current state. // -int DataLogger::ConfigureDailyFileName(const Event& evt) +int DataLogger::ConfigureNightlyFileName(const Event& evt) { if (evt.GetText() != NULL) { - fDailyFileName = std::string(evt.GetText()); - Message("New daily folder specified: " + fDailyFileName); + fNightlyFileName = std::string(evt.GetText()); + Message("New Nightly folder specified: " + fNightlyFileName); } else - Error("Empty daily folder given. Please specify a valid path."); + Error("Empty Nightly folder given. Please specify a valid path."); return GetCurrentState(); @@ -927,5 +1396,5 @@ } else - Error("Empty daily folder given. Please specify a valid path"); + Error("Empty Nightly folder given. Please specify a valid path"); return GetCurrentState(); @@ -942,4 +1411,7 @@ { fRunNumber = evt.GetInt(); + std::stringstream str; + str << "The new run number is: " << fRunNumber; + Message(str.str()); return GetCurrentState(); } @@ -951,51 +1423,70 @@ //! this is not a problem though because file are not opened so often. //! @ param type the type of the opened file. 0 = none open, 1 = log, 2 = text, 4 = fits -inline void DataLogger::NotifyOpenedFile(std::string name, int type, DimService* service) -{ - service->setQuality(type); - service->updateService(const_cast(name.c_str())); +inline void DataLogger::NotifyOpenedFile(std::string name, int type, DimDescribedService* service) +{ + if (fOpenedFilesIsOn) + { + if (fDebugIsOn) + { + stringstream str; + str << "Updating files service " << service->getName() << "with code: " << type << " and file: " << name; + Debug(str.str()); + str.str(""); + str << "Num subs: " << fNumSubAndFitsData.numSubscriptions << " Num open FITS: " << fNumSubAndFitsData.numOpenFits; + Debug(str.str()); + } + OpenFileToDim fToDim; + fToDim.code = type; + memcpy(fToDim.fileName, name.c_str(), name.size()+1); + service->setData(reinterpret_cast(&fToDim), name.size()+1+sizeof(int)); + service->setQuality(0); + service->updateService(); + } } // -------------------------------------------------------------------------- // //! Implements the Start transition. -//! Concatenates the given path for the daily file and the filename itself (based on the day), +//! Concatenates the given path for the Nightly file and the filename itself (based on the day), //! and tries to open it. //! @returns -//! kSM_DailyOpen if success, kSM_BadDailyConfig if failure +//! kSM_NightlyOpen if success, kSM_BadNightlyConfig if failure int DataLogger::StartPlease() { - //TODO concatenate the dailyFileName and the formatted date and extension to obtain the full file name + if (fDebugIsOn) + { + Debug("Starting..."); + } Time time; std::stringstream sTime; sTime << time.Y() << "_" << time.M() << "_" << time.D(); - fFullDailyLogFileName = fDailyFileName + '/' + sTime.str() + ".log"; - fDailyLogFile.open(fFullDailyLogFileName.c_str(), std::ios_base::out | std::ios_base::app); + fFullNightlyLogFileName = fNightlyFileName + '/' + sTime.str() + ".log"; + fNightlyLogFile.open(fFullNightlyLogFileName.c_str(), std::ios_base::out | std::ios_base::app); if (errno != 0) { std::stringstream str; - str << "Unable to open daily Log " << fFullDailyLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; + str << "Unable to open Nightly Log " << fFullNightlyLogFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; Error(str); } - fFullDailyReportFileName = fDailyFileName + '/' + sTime.str() + ".rep"; - fDailyReportFile.open(fFullDailyReportFileName.c_str(), std::ios_base::out | std::ios_base::app); + fFullNightlyReportFileName = fNightlyFileName + '/' + sTime.str() + ".rep"; + fNightlyReportFile.open(fFullNightlyReportFileName.c_str(), std::ios_base::out | std::ios_base::app); if (errno != 0) { std::stringstream str; - str << "Unable to open daily Report " << fFullDailyReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; + str << "Unable to open Nightly Report " << fFullNightlyReportFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; Error(str); } - if (!fDailyLogFile.is_open() || !fDailyReportFile.is_open()) + if (!fNightlyLogFile.is_open() || !fNightlyReportFile.is_open()) { //TODO send an error message - return kSM_BadDailyConfig; + return kSM_BadNightlyConfig; } //get the size of the newly opened file. struct stat st; - stat(fFullDailyLogFileName.c_str(), &st); - fBaseSizeDaily = st.st_size; - stat(fFullDailyReportFileName.c_str(), &st); - fBaseSizeDaily += st.st_size; + stat(fFullNightlyLogFileName.c_str(), &st); + fBaseSizeNightly = st.st_size; + stat(fFullNightlyReportFileName.c_str(), &st); + fBaseSizeNightly += st.st_size; fFileSizesMap.clear(); fBaseSizeRun = 0; @@ -1003,5 +1494,5 @@ //notify that files were opened std::string actualTargetDir; - if (fDailyFileName == ".") + if (fNightlyFileName == ".") { char currentPath[FILENAME_MAX]; @@ -1017,11 +1508,10 @@ else { - actualTargetDir = fDailyFileName; - } - - NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 3, fOpenedDailyFiles); + actualTargetDir = fNightlyFileName; + } + //notify that a new file has been opened. + NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 3, fOpenedNightlyFiles); - - return kSM_DailyOpen; + return kSM_NightlyOpen; } @@ -1046,8 +1536,8 @@ std::stringstream sTime; sTime << time.Y() << "_" << time.M() << "_" << time.D(); - //we open the dailyFile anyway, otherwise this function shouldn't have been called. - if (!sub.dailyFile.IsOpen()) - { - std::string partialName = fDailyFileName + '/' + sTime.str() + '_' + serviceName + ".fits"; + //we open the NightlyFile anyway, otherwise this function shouldn't have been called. + if (!sub.nightlyFile.IsOpen()) + { + std::string partialName = fNightlyFileName + '/' + sTime.str() + '_' + serviceName + ".fits"; AllocateFITSBuffers(sub); //get the size of the file we're about to open @@ -1056,11 +1546,11 @@ struct stat st; if (!stat(partialName.c_str(), &st)) - fBaseSizeDaily += st.st_size; + fBaseSizeNightly += st.st_size; fFileSizesMap[partialName] = 0; } - sub.dailyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, Out()); + sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, Out()); //notify the opening std::string actualTargetDir; - if (fDailyFileName == ".") + if (fNightlyFileName == ".") { char currentPath[FILENAME_MAX]; @@ -1070,15 +1560,22 @@ else { - actualTargetDir = fDailyFileName; + actualTargetDir = fNightlyFileName; } - NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 4, fOpenedDailyFiles); - fNumSubAndFits->updateService(); + NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 4, fOpenedNightlyFiles); + if (fNumSubAndFitsIsOn) + fNumSubAndFits->updateService(); + if (fDebugIsOn) + { + stringstream str; + str << "Opened Nightly FITS: " << partialName << " and table: FACT-" << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits; + Debug(str.str()); + } } if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging)) - {//buffer for the run file have already been allocated when doing the daily file + {//buffer for the run file have already been allocated when doing the Nightly file std::stringstream sRun; sRun << fRunNumber; #ifdef ONE_RUN_FITS_ONLY - std::string partialName = fRunFileName + '/' + sRun.str() + ".fits";//'_' + serviceName + ".fits"; + std::string partialName = fRunFileName + '/' + sRun.str() + ".fits"; if (fRunFitsFile == NULL) { @@ -1128,6 +1625,12 @@ sub.runFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, Out()); #endif //one_run_fits_only - fNumSubAndFits->updateService(); - + if (fNumSubAndFitsIsOn) + fNumSubAndFits->updateService(); + if (fDebugIsOn) + { + stringstream str; + str << "Opened Run FITS: " << partialName << " and table: FACT-" << serviceName << ".current number of opened FITS: " << fNumSubAndFitsData.numOpenFits; + Debug(str.str()); + } } } @@ -1140,9 +1643,9 @@ //Init the time columns of the file Description dateDesc(std::string("Time"), std::string("Modified Julian Date"), std::string("MjD")); - sub.dailyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double)); + sub.nightlyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double)); sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double)); Description QoSDesc("Qos", "Quality of service", "None"); - sub.dailyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int)); + sub.nightlyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int)); sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int)); @@ -1166,5 +1669,6 @@ {//TODO handle all the data format cases case 'c': - dataQualifier << "S"; + case 'C': + dataQualifier.str("S"); break; case 's': @@ -1172,14 +1676,18 @@ break; case 'i': + case 'I': dataQualifier << "J"; break; case 'l': + case 'L': dataQualifier << "J"; //TODO triple check that in FITS, long = int break; case 'f': + case 'F': dataQualifier << "E"; break; case 'd': + case 'D': dataQualifier << "D"; break; @@ -1196,10 +1704,11 @@ default: - Error("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 948."); + Error("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 1198."); }; - dataFormatsLocal.push_back(dataQualifier.str()); + //we skip the variable length strings for now (in fits only) + if (dataQualifier.str() != "S") + dataFormatsLocal.push_back(dataQualifier.str()); } - - sub.dailyFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size); + sub.nightlyFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size); sub.runFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size); } @@ -1210,9 +1719,21 @@ void DataLogger::WriteToFITS(SubscriptionType& sub) { - //dailyFile status (open or not) already checked - if (sub.dailyFile.IsOpen()) - sub.dailyFile.Write(sub.fConv); + //nightly File status (open or not) already checked + if (sub.nightlyFile.IsOpen()) + { + sub.nightlyFile.Write(sub.fConv); + if (fDebugIsOn) + { + Debug("Writing to nightly FITS " + sub.nightlyFile.fFileName); + } + } if (sub.runFile.IsOpen()) + { sub.runFile.Write(sub.fConv); + if (fDebugIsOn) + { + Debug("Writing to Run FITS " + sub.runFile.fFileName); + } + } } #endif //if has_fits @@ -1226,4 +1747,8 @@ int DataLogger::StartRunPlease() { + if (fDebugIsOn) + { + Debug("Starting Run Logging..."); + } //attempt to open run file with current parameters if (fRunNumber == -1) @@ -1311,4 +1836,8 @@ int DataLogger::StopRunPlease() { + if (fDebugIsOn) + { + Debug("Stopping Run Logging..."); + } if (!fRunLogFile.is_open() || !fRunReportFile.is_open()) return kSM_FatalError; @@ -1332,6 +1861,7 @@ #endif #endif - NotifyOpenedFile("None", 0, fOpenedRunFiles); - fNumSubAndFits->updateService(); + NotifyOpenedFile("", 0, fOpenedRunFiles); + if (fNumSubAndFitsIsOn) + fNumSubAndFits->updateService(); return kSM_WaitingRun; @@ -1345,8 +1875,12 @@ int DataLogger::GoToReadyPlease() { - if (fDailyLogFile.is_open()) - fDailyLogFile.close(); - if (fDailyReportFile.is_open()) - fDailyReportFile.close(); + if (fDebugIsOn) + { + Debug("Going to the Ready state..."); + } + if (fNightlyLogFile.is_open()) + fNightlyLogFile.close(); + if (fNightlyReportFile.is_open()) + fNightlyReportFile.close(); if (fRunLogFile.is_open()) @@ -1359,6 +1893,6 @@ for (std::map::iterator j = i->second.begin(); j != i->second.end(); j++) { - if (j->second.dailyFile.IsOpen()) - j->second.dailyFile.Close(); + if (j->second.nightlyFile.IsOpen()) + j->second.nightlyFile.Close(); if (j->second.runFile.IsOpen()) j->second.runFile.Close(); @@ -1374,11 +1908,12 @@ #endif if (GetCurrentState() == kSM_Logging) - NotifyOpenedFile("None", 0, fOpenedRunFiles); + NotifyOpenedFile("", 0, fOpenedRunFiles); if (GetCurrentState() == kSM_Logging || GetCurrentState() == kSM_WaitingRun || - GetCurrentState() == kSM_DailyOpen) + GetCurrentState() == kSM_NightlyOpen) { - NotifyOpenedFile("None", 0, fOpenedDailyFiles); - fNumSubAndFits->updateService(); + NotifyOpenedFile("", 0, fOpenedNightlyFiles); + if (fNumSubAndFitsIsOn) + fNumSubAndFits->updateService(); } return kSM_Ready; @@ -1390,7 +1925,30 @@ //! @returns //! kSM_WaitingRun -int DataLogger::DailyToWaitRunPlease() -{ +int DataLogger::NightlyToWaitRunPlease() +{ + if (fDebugIsOn) + { + Debug("Going to Wait Run Number state..."); + } return kSM_WaitingRun; +} + +void DataLogger::setBlackWhiteList(const std::string& black, bool isBlack) +{ + if (fDebugIsOn) + { + if (isBlack) + Debug("Setting BLACK list: " + black); + else + Debug("Setting WHITE list: " + black); + } + fGreyList.clear(); + stringstream stream(black); + + string buffer; + while (getline(stream, buffer, '|')) { + fGreyList.insert(buffer); + } + fIsBlackList = isBlack; } @@ -1436,9 +1994,13 @@ DataLogger logger(wout); -// if (conf.Has("black-list")) -// logger.setBlackList(conf.Get("black-list")); -// else if (conf.Has("white-list")) -// logger.setWhiteList(conf.Get("white-list")); - + if (conf.Has("black-list")) + { if (conf.Get("black-list") != "") + logger.setBlackWhiteList(conf.Get("black-list"), true); + else if (conf.Has("white-list")) + { + if (conf.Get("white-list") != "") + logger.setBlackWhiteList(conf.Get("white-list"), false); + } + } shell.SetReceiver(logger);