Index: trunk/FACT++/src/dataLogger.cc =================================================================== --- trunk/FACT++/src/dataLogger.cc (revision 10735) +++ trunk/FACT++/src/dataLogger.cc (revision 10739) @@ -66,7 +66,4 @@ #include -//#define HAVE_FITS -//#define ONE_RUN_FITS_ONLY - #include @@ -110,5 +107,5 @@ void setQuality(int){}; }; -class DataLogger : public StateMachineDim, DimInfoHandler//, DimServiceInfoList //,DimInfoHandler +class DataLogger : public StateMachineDim, DimInfoHandler { public: @@ -287,5 +284,5 @@ #ifdef ONE_RUN_FITS_ONLY ///FITS file for runs. only one, hence dealt with in the dataLogger itself - FITS* fRunFitsFile; + CCfits::FITS* fRunFitsFile; #endif //one_run_fits_only #endif//has_fits @@ -317,11 +314,13 @@ inline void NotifyOpenedFile(std::string name, int type, DimDescribedService* service); + public: -// void setBlackWhiteList(const std::string& , bool); bool SetConfiguration(Configuration& conf); private: - std::set fGreyList; - bool fIsBlackList; + std::set fBlackList; + std::set fWhiteList; + bool fHasBlackList; + bool fHasWhiteList; bool fDebugIsOn; float fStatsPeriodDuration; @@ -338,4 +337,10 @@ ///Small function for calculating the total size written so far void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting); + + ///vectors to keep track of opened Fits files, for grouping purposes. + //This cannot be done otherwise, as services may disapear before the files are nicely closed. Hence which files were opened must be remembered. + map fOpenedRunFits; + map fOpenedNightlyFits; + void CreateFitsGrouping(bool runGroup); }; //DataLogger @@ -444,7 +449,4 @@ void DataLogger::ServicesMonitoring() { - //create the DIM service -// int dataSize = 2*sizeof(long) + sizeof(long); - DataLoggerStats statVar; statVar.sizeWritten = 0; @@ -472,70 +474,5 @@ sleep(fStatsPeriodDuration); //update the fits files sizes - calculateTotalSizeWritten(statVar, statWarning, false);/* -#ifdef HAVE_FITS - SubscriptionsListType::iterator x; - std::map::iterator y; - bool runFileDone = false; - for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++) - { - for (y=x->second.begin(); y != x->second.end(); y++) - { - if (y->second.runFile.IsOpen() && !runFileDone) - { - fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); -#ifdef ONE_FITS_ONLY - runFileDone = true; -#endif - } - if (y->second.nightlyFile.IsOpen()) - fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); - } - } -#endif - struct stat st; - //gather log and report files sizes on disk - if (fNightlyLogFile.is_open()) - { - stat(fFullNightlyLogFileName.c_str(), &st); - fFileSizesMap[fFullNightlyLogFileName] = st.st_size; - } - if (fNightlyReportFile.is_open()) - { - stat(fFullNightlyReportFileName.c_str(), &st); - fFileSizesMap[fFullNightlyReportFileName] = st.st_size; - } - if (fRunLogFile.is_open()) - { - stat(fFullRunLogFileName.c_str(), &st); - fFileSizesMap[fFullRunLogFileName] = st.st_size; - } - if (fRunReportFile.is_open()) - { - stat(fFullRunReportFileName.c_str(), &st); - fFileSizesMap[fFullRunReportFileName] = st.st_size; - } - - if (!statvfs(fNightlyFileName.c_str(), &vfs)) - { - statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; - statWarning = false; - } - else - { - std::stringstream str; - str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; - if (!statWarning) - Error(str); - statWarning = true; - statVar.freeSpace = -1; - } - - //sum up all the file sizes. past and present - statVar.sizeWritten = 0; - for (std::map::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) - statVar.sizeWritten += it->second; - statVar.sizeWritten -= fBaseSizeNightly; - statVar.sizeWritten -= fBaseSizeRun; - */ + calculateTotalSizeWritten(statVar, statWarning, false); if (fStatsPeriodDuration == 0.0f) continue; @@ -570,8 +507,7 @@ dic_disable_padding(); dis_disable_padding(); - //initialize member data - fNightlyFileName = ".";//"/home/lyard/log";// - fRunFileName = ".";//"/home/lyard/log"; + fNightlyFileName = "."; + fRunFileName = "."; fRunNumber = -1; fPreviousRunNumber = fRunNumber; @@ -677,8 +613,9 @@ "|NumOpenFiles[int]:number of files currently open by the data logger"); - //black/white list - fIsBlackList = true; - fGreyList.clear(); - + //black/white list + fHasBlackList = false; + fHasWhiteList = false; + fBlackList.clear(); + fWhiteList.clear(); //services parameters fDebugIsOn = false; @@ -750,6 +687,4 @@ if ((i->find("DIS_DNS") != std::string::npos) || (i->find("DATA_LOGGER") != std::string::npos)) - continue; - if (fIsBlackList && (fGreyList.find(*i) != fGreyList.end())) continue; //find the current server in our subscription list @@ -781,15 +716,15 @@ if (*givenSubs == "SERVICE_LIST") continue; - if (fIsBlackList && fGreyList.find(*givenSubs) != fGreyList.end()) - continue; - - if (fIsBlackList && fGreyList.find((*i) + "/" + (*givenSubs)) != fGreyList.end()) - continue; - else if (!fIsBlackList && - (fGreyList.find((*i) + "/" + (*givenSubs)) == fGreyList.end()) && - (fGreyList.find(*i) == fGreyList.end()) && - (fGreyList.find(*givenSubs) == fGreyList.end())) - continue; - if (cSubs->second.find(*givenSubs) == cSubs->second.end()) + + if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) && + (fWhiteList.find(*i + "/" + *givenSubs) == fWhiteList.end()) && + (fWhiteList.find("/" + *givenSubs) == fWhiteList.end())) + continue; + if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) || + (fBlackList.find(*i + "/" + *givenSubs) != fBlackList.end()) || + (fBlackList.find("/" + *givenSubs) != fBlackList.end()))) + continue; + + if (cSubs->second.find(*givenSubs) == cSubs->second.end()) {//service not found. Add it cSubs->second[*givenSubs].dimInfo = new DimStampedInfo(((*i) + "/" + *givenSubs).c_str(), const_cast(""), this); @@ -812,14 +747,13 @@ if (*j == "SERVICE_LIST") continue; - if (fIsBlackList && fGreyList.find(*j) != fGreyList.end()) - continue; - - if (fIsBlackList && fGreyList.find((*i) + "/" + (*j)) != fGreyList.end()) - continue; - else if (!fIsBlackList && - (fGreyList.find((*i) + "/" + (*j)) == fGreyList.end()) && - (fGreyList.find(*i) == fGreyList.end()) && - (fGreyList.find(*j) == fGreyList.end())) - continue; + if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) && + (fWhiteList.find(*i + "/" + *j) == fWhiteList.end()) && + (fWhiteList.find("/" + *j) == fWhiteList.end())) + continue; + if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) || + (fBlackList.find(*i + "/" + *j) != fBlackList.end()) || + (fBlackList.find("/" + *j) != fBlackList.end()))) + continue; + liste[*j].dimInfo = new DimStampedInfo(((*i) + "/" + (*j)).c_str(), const_cast(""), this); @@ -1294,77 +1228,5 @@ bool statWarning = false; DataLoggerStats statVar; - calculateTotalSizeWritten(statVar, statWarning, false);/* -#ifdef HAVE_FITS - str.str(""); - str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:"; - Message(str.str()); - SubscriptionsListType::iterator x; - std::map::iterator y; - bool runFileDone = false; - for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++) - { - for (y=x->second.begin(); y != x->second.end(); y++) - { - if (y->second.runFile.IsOpen() && !runFileDone) - { - fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); - Message("-> "+y->second.runFile.fFileName); -#ifdef ONE_FITS_ONLY - runFileDone = true; -#endif - } - if (y->second.nightlyFile.IsOpen()) - { - fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); - Message("-> "+y->second.nightlyFile.fFileName); - } - } - } -#else - Message("FITS output disabled at compilation"); -#endif - struct stat st; - DataLoggerStats statVar; - //gather log and report files sizes on disk - if (fNightlyLogFile.is_open()) - { - stat(fFullNightlyLogFileName.c_str(), &st); - fFileSizesMap[fFullNightlyLogFileName] = st.st_size; - } - if (fNightlyReportFile.is_open()) - { - stat(fFullNightlyReportFileName.c_str(), &st); - fFileSizesMap[fFullNightlyReportFileName] = st.st_size; - } - if (fRunLogFile.is_open()) - { - stat(fFullRunLogFileName.c_str(), &st); - fFileSizesMap[fFullRunLogFileName] = st.st_size; - } - if (fRunReportFile.is_open()) - { - stat(fFullRunReportFileName.c_str(), &st); - fFileSizesMap[fFullRunReportFileName] = st.st_size; - } - struct statvfs vfs; - if (!statvfs(fNightlyFileName.c_str(), &vfs)) - { - statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; - } - else - { - str.str(""); - str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; - Error(str);; - statVar.freeSpace = -1; - } - - //sum up all the file sizes. past and present - statVar.sizeWritten = 0; - for (std::map::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) - statVar.sizeWritten += it->second; - statVar.sizeWritten -= fBaseSizeNightly; - statVar.sizeWritten -= fBaseSizeRun; - */ + calculateTotalSizeWritten(statVar, statWarning, false); Message("---------------- STATS ------------------"); str.str(""); @@ -1392,14 +1254,16 @@ Message(" -> "+it2->first); } - if (fIsBlackList) - Message("------------- BLOCK LIST ----------------"); - else - if (fGreyList.size() != 0) + if (fHasBlackList) + { + Message("------------- BLOCK LIST ----------------"); + for (set::iterator it=fBlackList.begin(); it != fBlackList.end(); it++) + Message(*it); + } + if (fHasWhiteList) + { Message("----------- ALLOW LIST ------------------"); - for (set::iterator it=fGreyList.begin(); it != fGreyList.end(); it++) - Message(*it); - - if (fGreyList.size()==0) - Message(" "); + for (set::iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++) + Message(*it); + } return GetCurrentState(); @@ -1665,4 +1529,6 @@ //notify that a new file has been opened. NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 3, fOpenedNightlyFiles); + + fOpenedNightlyFits.clear(); return kSM_NightlyOpen; @@ -1692,5 +1558,6 @@ if (!sub.nightlyFile.IsOpen()) { - std::string partialName = fNightlyFileName + '/' + sTime.str() + '_' + serviceName + ".fits"; + std::string fileNameOnly = sTime.str() + '_' + serviceName + ".fits"; + std::string partialName = fNightlyFileName + '/' + fileNameOnly; AllocateFITSBuffers(sub); //get the size of the file we're about to open @@ -1701,6 +1568,9 @@ fBaseSizeNightly += st.st_size; fFileSizesMap[partialName] = 0; + //remember that this file was opened. + fOpenedNightlyFits[fileNameOnly] = serviceName; } sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out()); + //notify the opening std::string actualTargetDir; @@ -1730,9 +1600,11 @@ sRun << fRunNumber; #ifdef ONE_RUN_FITS_ONLY - std::string partialName = fRunFileName + '/' + sRun.str() + ".fits"; + std::string fileNameOnly = sRun.str() + ".fits"; + std::string partialName = fRunFileName + '/' + fileNameOnly; if (fRunFitsFile == NULL) { #else - std::string partialName = fRunFileName + '/' + sRun.str() + '_' + serviceName + ".fits"; + std::string fileNameOnly = sRun.str() + '_' + serviceName + ".fits"; + std::string partialName = fRunFileName + '/' + fileNameOnly; #endif //get the size of the file we're about to open @@ -1745,9 +1617,10 @@ fBaseSizeRun = 0; fFileSizesMap[partialName] = 0; + fOpenedRunFits[fileNameOnly] = serviceName; } #ifdef ONE_RUN_FITS_ONLY try { - fRunFitsFile = new FITS(partialName, RWmode::Write); + fRunFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write); (fNumSubAndFitsData.numOpenFits)++; } @@ -1981,6 +1854,95 @@ NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 3, fOpenedRunFiles); + fOpenedRunFits.clear(); + return kSM_Logging; } +#ifdef HAVE_FITS +void DataLogger::CreateFitsGrouping(bool runGroup) +{ + //create the FITS group corresponding to the ending run.ETIENNEICI + CCfits::FITS* groupFile; + +#ifdef ONE_RUN_FITS_ONLY + if (runGroup) + return; +#endif + map& filesToGroup = runGroup? fOpenedRunFits : fOpenedNightlyFits; + unsigned int numFilesToGroup = filesToGroup.size(); + if (numFilesToGroup <= 1) + { + filesToGroup.clear(); + return; + } + stringstream groupName; + if (runGroup) + { + groupName << fRunFileName << '/' << fRunNumber << ".fits"; + } + else + { + Time time; + std::stringstream sTime; + sTime << time.Y() << "_" << time.M() << "_" << time.D(); + + groupName << fNightlyFileName << '/' << sTime.str() << ".fits"; + } + try + { + groupFile = new CCfits::FITS(groupName.str(), CCfits::RWmode::Write); + //setup the column names + int maxCharLength = FILENAME_MAX; + stringstream pathTypeName; + pathTypeName << maxCharLength << "A"; + vector names; + vector dataTypes; + names.push_back("MEMBER_XTENSION"); + dataTypes.push_back("8A"); + names.push_back("MEMBER_URI_TYPE"); + dataTypes.push_back("3A"); + names.push_back("MEMBER_LOCATION"); + dataTypes.push_back(pathTypeName.str()); + names.push_back("MEMBER_NAME"); + dataTypes.push_back(pathTypeName.str()); + CCfits::Table* groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes); + } + catch (CCfits::FitsError e) + { + std::stringstream str; + str << "Could not open or create FITS table GROUPING in file " << groupName.str() << " reason: " << e.message(); + Error(str); + return; + } + + //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about. + //use cfitsio routines instead + groupTable->makeThisCurrent(); + //create appropriate buffer. + unsigned char* fitsBuffer = new unsigned char[8 + 3 + 2*maxCharLength + 1]; //+1 for trailling character + char* startOfExtension = reinterpret_cast(fitsBuffer); + char* startOfURI = reinterpret_cast(&fitsBuffer[8]); + char* startOfLocation = reinterpret_cast(&fitsBuffer[8 + 3]); + char* startOfName = reinterpret_cast(&fitsBuffer[8+3+maxCharLength]); + sprintf(startOfExtension, "%s", "BINTABLE"); + sprintf(startOfURI, "%s", "URL"); + int i=1; + for (map::iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++, i++) + { + strcpy(startOfLocation, it->first.c_str()); + strcpy(startOfName, it->second.c_str()); + int status = 0; + fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status); + if (status) + { + stringstream str; + str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status; + Error(str.str()); + } + } + + filesToGroup.clear(); + delete groupFile; +} +#endif //HAVE_FITS // -------------------------------------------------------------------------- // @@ -1991,4 +1953,5 @@ int DataLogger::StopRunPlease() { + if (fDebugIsOn) { @@ -2020,4 +1983,7 @@ if (fNumSubAndFitsIsOn) fNumSubAndFits->updateService(); + + CreateFitsGrouping(true); + return kSM_WaitingRun; @@ -2073,4 +2039,7 @@ fNumSubAndFits->updateService(); } + CreateFitsGrouping(true); + CreateFitsGrouping(false); + return kSM_Ready; } @@ -2095,5 +2064,6 @@ //Set the block or allow list - fGreyList.clear(); + fBlackList.clear(); + fWhiteList.clear(); if (conf.Has("block")) { @@ -2101,48 +2071,34 @@ if (vec.size() != 0) { - fIsBlackList = true; + fHasBlackList = true; if (fDebugIsOn) Debug("Setting BLOCK list:"); } - else if (conf.Has("allow")) + for (vector::iterator it = vec.begin(); it != vec.end(); it++) { - vec = conf.Get>("allow"); - if (vec.size() != 0) - { - fIsBlackList = false; - if (fDebugIsOn) - Debug("Setting ALLOW list:"); - } + fBlackList.insert(*it); + if (fDebugIsOn) + Debug(" " + *it); + } + } + if (conf.Has("allow")) + { + vector vec = conf.Get>("allow"); + if (vec.size() != 0) + { + fHasWhiteList = true; + if (fDebugIsOn) + Debug("Setting ALLOW list:"); } for (vector::iterator it=vec.begin(); it != vec.end(); it++) { - fGreyList.insert(*it); + fWhiteList.insert(*it); if (fDebugIsOn) Debug(" " + *it); } } - return true; } -/* -void DataLogger::setBlackWhiteList(const std::string& black, bool isBlack) -{ - if (fDebugIsOn) - { - if (isBlack) - Debug("Setting BLACK list: " + black); - else - Debug("Setting WHITE list: " + black); - } - fGreyList.clear(); - stringstream stream(black); - - string buffer; - while (getline(stream, buffer, '|')) { - fGreyList.insert(buffer); - } - fIsBlackList = isBlack; -} -*/ + // --------------------------------------------------------------------------