Changeset 10739 for trunk/FACT++
- Timestamp:
- 05/18/11 12:01:15 (14 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/FACT++/src/dataLogger.cc
r10725 r10739 66 66 #include <sys/stat.h> 67 67 68 //#define HAVE_FITS69 //#define ONE_RUN_FITS_ONLY70 71 68 #include <fstream> 72 69 … … 110 107 void setQuality(int){}; 111 108 }; 112 class DataLogger : public StateMachineDim, DimInfoHandler //, DimServiceInfoList //,DimInfoHandler109 class DataLogger : public StateMachineDim, DimInfoHandler 113 110 { 114 111 public: … … 287 284 #ifdef ONE_RUN_FITS_ONLY 288 285 ///FITS file for runs. only one, hence dealt with in the dataLogger itself 289 FITS* fRunFitsFile;286 CCfits::FITS* fRunFitsFile; 290 287 #endif //one_run_fits_only 291 288 #endif//has_fits … … 317 314 318 315 inline void NotifyOpenedFile(std::string name, int type, DimDescribedService* service); 316 319 317 public: 320 // void setBlackWhiteList(const std::string& , bool);321 318 bool SetConfiguration(Configuration& conf); 322 319 323 320 private: 324 std::set<std::string> fGreyList; 325 bool fIsBlackList; 321 std::set<std::string> fBlackList; 322 std::set<std::string> fWhiteList; 323 bool fHasBlackList; 324 bool fHasWhiteList; 326 325 bool fDebugIsOn; 327 326 float fStatsPeriodDuration; … … 338 337 ///Small function for calculating the total size written so far 339 338 void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting); 339 340 ///vectors to keep track of opened Fits files, for grouping purposes. 341 //This cannot be done otherwise, as services may disapear before the files are nicely closed. Hence which files were opened must be remembered. 342 map<string, string> fOpenedRunFits; 343 map<string, string> fOpenedNightlyFits; 344 void CreateFitsGrouping(bool runGroup); 340 345 }; //DataLogger 341 346 … … 444 449 void DataLogger::ServicesMonitoring() 445 450 { 446 //create the DIM service447 // int dataSize = 2*sizeof(long) + sizeof(long);448 449 451 DataLoggerStats statVar; 450 452 statVar.sizeWritten = 0; … … 472 474 sleep(fStatsPeriodDuration); 473 475 //update the fits files sizes 474 calculateTotalSizeWritten(statVar, statWarning, false);/* 475 #ifdef HAVE_FITS 476 SubscriptionsListType::iterator x; 477 std::map<std::string, SubscriptionType>::iterator y; 478 bool runFileDone = false; 479 for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++) 480 { 481 for (y=x->second.begin(); y != x->second.end(); y++) 482 { 483 if (y->second.runFile.IsOpen() && !runFileDone) 484 { 485 fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); 486 #ifdef ONE_FITS_ONLY 487 runFileDone = true; 488 #endif 489 } 490 if (y->second.nightlyFile.IsOpen()) 491 fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); 492 } 493 } 494 #endif 495 struct stat st; 496 //gather log and report files sizes on disk 497 if (fNightlyLogFile.is_open()) 498 { 499 stat(fFullNightlyLogFileName.c_str(), &st); 500 fFileSizesMap[fFullNightlyLogFileName] = st.st_size; 501 } 502 if (fNightlyReportFile.is_open()) 503 { 504 stat(fFullNightlyReportFileName.c_str(), &st); 505 fFileSizesMap[fFullNightlyReportFileName] = st.st_size; 506 } 507 if (fRunLogFile.is_open()) 508 { 509 stat(fFullRunLogFileName.c_str(), &st); 510 fFileSizesMap[fFullRunLogFileName] = st.st_size; 511 } 512 if (fRunReportFile.is_open()) 513 { 514 stat(fFullRunReportFileName.c_str(), &st); 515 fFileSizesMap[fFullRunReportFileName] = st.st_size; 516 } 517 518 if (!statvfs(fNightlyFileName.c_str(), &vfs)) 519 { 520 statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; 521 statWarning = false; 522 } 523 else 524 { 525 std::stringstream str; 526 str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; 527 if (!statWarning) 528 Error(str); 529 statWarning = true; 530 statVar.freeSpace = -1; 531 } 532 533 //sum up all the file sizes. past and present 534 statVar.sizeWritten = 0; 535 for (std::map<std::string, long>::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) 536 statVar.sizeWritten += it->second; 537 statVar.sizeWritten -= fBaseSizeNightly; 538 statVar.sizeWritten -= fBaseSizeRun; 539 */ 476 calculateTotalSizeWritten(statVar, statWarning, false); 540 477 if (fStatsPeriodDuration == 0.0f) 541 478 continue; … … 570 507 dic_disable_padding(); 571 508 dis_disable_padding(); 572 573 509 //initialize member data 574 fNightlyFileName = "."; //"/home/lyard/log";//575 fRunFileName = "."; //"/home/lyard/log";510 fNightlyFileName = "."; 511 fRunFileName = "."; 576 512 fRunNumber = -1; 577 513 fPreviousRunNumber = fRunNumber; … … 677 613 "|NumOpenFiles[int]:number of files currently open by the data logger"); 678 614 679 //black/white list 680 fIsBlackList = true; 681 fGreyList.clear(); 682 615 //black/white list 616 fHasBlackList = false; 617 fHasWhiteList = false; 618 fBlackList.clear(); 619 fWhiteList.clear(); 683 620 //services parameters 684 621 fDebugIsOn = false; … … 750 687 if ((i->find("DIS_DNS") != std::string::npos) || 751 688 (i->find("DATA_LOGGER") != std::string::npos)) 752 continue;753 if (fIsBlackList && (fGreyList.find(*i) != fGreyList.end()))754 689 continue; 755 690 //find the current server in our subscription list … … 781 716 if (*givenSubs == "SERVICE_LIST") 782 717 continue; 783 if (fIsBlackList && fGreyList.find(*givenSubs) != fGreyList.end()) 784 continue;785 786 if (fIsBlackList && fGreyList.find((*i) + "/" + (*givenSubs)) != fGreyList.end())787 788 else if (!fIsBlackList && 789 (fGreyList.find((*i) + "/" + (*givenSubs)) == fGreyList.end()) && 790 (fGreyList.find(*i) == fGreyList.end()) && 791 (fGreyList.find(*givenSubs) == fGreyList.end())) 792 continue; 793 718 719 if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) && 720 (fWhiteList.find(*i + "/" + *givenSubs) == fWhiteList.end()) && 721 (fWhiteList.find("/" + *givenSubs) == fWhiteList.end())) 722 continue; 723 if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) || 724 (fBlackList.find(*i + "/" + *givenSubs) != fBlackList.end()) || 725 (fBlackList.find("/" + *givenSubs) != fBlackList.end()))) 726 continue; 727 728 if (cSubs->second.find(*givenSubs) == cSubs->second.end()) 794 729 {//service not found. Add it 795 730 cSubs->second[*givenSubs].dimInfo = new DimStampedInfo(((*i) + "/" + *givenSubs).c_str(), const_cast<char*>(""), this); … … 812 747 if (*j == "SERVICE_LIST") 813 748 continue; 814 if (fIsBlackList && fGreyList.find(*j) != fGreyList.end()) 815 continue; 816 817 if (fIsBlackList && fGreyList.find((*i) + "/" + (*j)) != fGreyList.end()) 818 continue; 819 else if (!fIsBlackList && 820 (fGreyList.find((*i) + "/" + (*j)) == fGreyList.end()) && 821 (fGreyList.find(*i) == fGreyList.end()) && 822 (fGreyList.find(*j) == fGreyList.end())) 823 continue; 749 if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) && 750 (fWhiteList.find(*i + "/" + *j) == fWhiteList.end()) && 751 (fWhiteList.find("/" + *j) == fWhiteList.end())) 752 continue; 753 if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) || 754 (fBlackList.find(*i + "/" + *j) != fBlackList.end()) || 755 (fBlackList.find("/" + *j) != fBlackList.end()))) 756 continue; 757 824 758 825 759 liste[*j].dimInfo = new DimStampedInfo(((*i) + "/" + (*j)).c_str(), const_cast<char*>(""), this); … … 1294 1228 bool statWarning = false; 1295 1229 DataLoggerStats statVar; 1296 calculateTotalSizeWritten(statVar, statWarning, false);/* 1297 #ifdef HAVE_FITS 1298 str.str(""); 1299 str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:"; 1300 Message(str.str()); 1301 SubscriptionsListType::iterator x; 1302 std::map<std::string, SubscriptionType>::iterator y; 1303 bool runFileDone = false; 1304 for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++) 1305 { 1306 for (y=x->second.begin(); y != x->second.end(); y++) 1307 { 1308 if (y->second.runFile.IsOpen() && !runFileDone) 1309 { 1310 fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize(); 1311 Message("-> "+y->second.runFile.fFileName); 1312 #ifdef ONE_FITS_ONLY 1313 runFileDone = true; 1314 #endif 1315 } 1316 if (y->second.nightlyFile.IsOpen()) 1317 { 1318 fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize(); 1319 Message("-> "+y->second.nightlyFile.fFileName); 1320 } 1321 } 1322 } 1323 #else 1324 Message("FITS output disabled at compilation"); 1325 #endif 1326 struct stat st; 1327 DataLoggerStats statVar; 1328 //gather log and report files sizes on disk 1329 if (fNightlyLogFile.is_open()) 1330 { 1331 stat(fFullNightlyLogFileName.c_str(), &st); 1332 fFileSizesMap[fFullNightlyLogFileName] = st.st_size; 1333 } 1334 if (fNightlyReportFile.is_open()) 1335 { 1336 stat(fFullNightlyReportFileName.c_str(), &st); 1337 fFileSizesMap[fFullNightlyReportFileName] = st.st_size; 1338 } 1339 if (fRunLogFile.is_open()) 1340 { 1341 stat(fFullRunLogFileName.c_str(), &st); 1342 fFileSizesMap[fFullRunLogFileName] = st.st_size; 1343 } 1344 if (fRunReportFile.is_open()) 1345 { 1346 stat(fFullRunReportFileName.c_str(), &st); 1347 fFileSizesMap[fFullRunReportFileName] = st.st_size; 1348 } 1349 struct statvfs vfs; 1350 if (!statvfs(fNightlyFileName.c_str(), &vfs)) 1351 { 1352 statVar.freeSpace = vfs.f_bsize*vfs.f_bavail; 1353 } 1354 else 1355 { 1356 str.str(""); 1357 str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]"; 1358 Error(str);; 1359 statVar.freeSpace = -1; 1360 } 1361 1362 //sum up all the file sizes. past and present 1363 statVar.sizeWritten = 0; 1364 for (std::map<std::string, long>::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end(); it++) 1365 statVar.sizeWritten += it->second; 1366 statVar.sizeWritten -= fBaseSizeNightly; 1367 statVar.sizeWritten -= fBaseSizeRun; 1368 */ 1230 calculateTotalSizeWritten(statVar, statWarning, false); 1369 1231 Message("---------------- STATS ------------------"); 1370 1232 str.str(""); … … 1392 1254 Message(" -> "+it2->first); 1393 1255 } 1394 if (fIsBlackList) 1395 Message("------------- BLOCK LIST ----------------"); 1396 else 1397 if (fGreyList.size() != 0) 1256 if (fHasBlackList) 1257 { 1258 Message("------------- BLOCK LIST ----------------"); 1259 for (set<string>::iterator it=fBlackList.begin(); it != fBlackList.end(); it++) 1260 Message(*it); 1261 } 1262 if (fHasWhiteList) 1263 { 1398 1264 Message("----------- ALLOW LIST ------------------"); 1399 for (set<string>::iterator it=fGreyList.begin(); it != fGreyList.end(); it++) 1400 Message(*it); 1401 1402 if (fGreyList.size()==0) 1403 Message(" <empty>"); 1265 for (set<string>::iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++) 1266 Message(*it); 1267 } 1404 1268 1405 1269 return GetCurrentState(); … … 1665 1529 //notify that a new file has been opened. 1666 1530 NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 3, fOpenedNightlyFiles); 1531 1532 fOpenedNightlyFits.clear(); 1667 1533 1668 1534 return kSM_NightlyOpen; … … 1692 1558 if (!sub.nightlyFile.IsOpen()) 1693 1559 { 1694 std::string partialName = fNightlyFileName + '/' + sTime.str() + '_' + serviceName + ".fits"; 1560 std::string fileNameOnly = sTime.str() + '_' + serviceName + ".fits"; 1561 std::string partialName = fNightlyFileName + '/' + fileNameOnly; 1695 1562 AllocateFITSBuffers(sub); 1696 1563 //get the size of the file we're about to open … … 1701 1568 fBaseSizeNightly += st.st_size; 1702 1569 fFileSizesMap[partialName] = 0; 1570 //remember that this file was opened. 1571 fOpenedNightlyFits[fileNameOnly] = serviceName; 1703 1572 } 1704 1573 sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out()); 1574 1705 1575 //notify the opening 1706 1576 std::string actualTargetDir; … … 1730 1600 sRun << fRunNumber; 1731 1601 #ifdef ONE_RUN_FITS_ONLY 1732 std::string partialName = fRunFileName + '/' + sRun.str() + ".fits"; 1602 std::string fileNameOnly = sRun.str() + ".fits"; 1603 std::string partialName = fRunFileName + '/' + fileNameOnly; 1733 1604 if (fRunFitsFile == NULL) 1734 1605 { 1735 1606 #else 1736 std::string partialName = fRunFileName + '/' + sRun.str() + '_' + serviceName + ".fits"; 1607 std::string fileNameOnly = sRun.str() + '_' + serviceName + ".fits"; 1608 std::string partialName = fRunFileName + '/' + fileNameOnly; 1737 1609 #endif 1738 1610 //get the size of the file we're about to open … … 1745 1617 fBaseSizeRun = 0; 1746 1618 fFileSizesMap[partialName] = 0; 1619 fOpenedRunFits[fileNameOnly] = serviceName; 1747 1620 } 1748 1621 #ifdef ONE_RUN_FITS_ONLY 1749 1622 try 1750 1623 { 1751 fRunFitsFile = new FITS(partialName, RWmode::Write);1624 fRunFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write); 1752 1625 (fNumSubAndFitsData.numOpenFits)++; 1753 1626 } … … 1981 1854 NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 3, fOpenedRunFiles); 1982 1855 1856 fOpenedRunFits.clear(); 1857 1983 1858 return kSM_Logging; 1984 1859 } 1860 #ifdef HAVE_FITS 1861 void DataLogger::CreateFitsGrouping(bool runGroup) 1862 { 1863 //create the FITS group corresponding to the ending run.ETIENNEICI 1864 CCfits::FITS* groupFile; 1865 1866 #ifdef ONE_RUN_FITS_ONLY 1867 if (runGroup) 1868 return; 1869 #endif 1870 map<string, string>& filesToGroup = runGroup? fOpenedRunFits : fOpenedNightlyFits; 1871 unsigned int numFilesToGroup = filesToGroup.size(); 1872 if (numFilesToGroup <= 1) 1873 { 1874 filesToGroup.clear(); 1875 return; 1876 } 1877 stringstream groupName; 1878 if (runGroup) 1879 { 1880 groupName << fRunFileName << '/' << fRunNumber << ".fits"; 1881 } 1882 else 1883 { 1884 Time time; 1885 std::stringstream sTime; 1886 sTime << time.Y() << "_" << time.M() << "_" << time.D(); 1887 1888 groupName << fNightlyFileName << '/' << sTime.str() << ".fits"; 1889 } 1890 try 1891 { 1892 groupFile = new CCfits::FITS(groupName.str(), CCfits::RWmode::Write); 1893 //setup the column names 1894 int maxCharLength = FILENAME_MAX; 1895 stringstream pathTypeName; 1896 pathTypeName << maxCharLength << "A"; 1897 vector<string> names; 1898 vector<string> dataTypes; 1899 names.push_back("MEMBER_XTENSION"); 1900 dataTypes.push_back("8A"); 1901 names.push_back("MEMBER_URI_TYPE"); 1902 dataTypes.push_back("3A"); 1903 names.push_back("MEMBER_LOCATION"); 1904 dataTypes.push_back(pathTypeName.str()); 1905 names.push_back("MEMBER_NAME"); 1906 dataTypes.push_back(pathTypeName.str()); 1907 CCfits::Table* groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes); 1908 } 1909 catch (CCfits::FitsError e) 1910 { 1911 std::stringstream str; 1912 str << "Could not open or create FITS table GROUPING in file " << groupName.str() << " reason: " << e.message(); 1913 Error(str); 1914 return; 1915 } 1916 1917 //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about. 1918 //use cfitsio routines instead 1919 groupTable->makeThisCurrent(); 1920 //create appropriate buffer. 1921 unsigned char* fitsBuffer = new unsigned char[8 + 3 + 2*maxCharLength + 1]; //+1 for trailling character 1922 char* startOfExtension = reinterpret_cast<char*>(fitsBuffer); 1923 char* startOfURI = reinterpret_cast<char*>(&fitsBuffer[8]); 1924 char* startOfLocation = reinterpret_cast<char*>(&fitsBuffer[8 + 3]); 1925 char* startOfName = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]); 1926 sprintf(startOfExtension, "%s", "BINTABLE"); 1927 sprintf(startOfURI, "%s", "URL"); 1928 int i=1; 1929 for (map<string, string>::iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++, i++) 1930 { 1931 strcpy(startOfLocation, it->first.c_str()); 1932 strcpy(startOfName, it->second.c_str()); 1933 int status = 0; 1934 fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status); 1935 if (status) 1936 { 1937 stringstream str; 1938 str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status; 1939 Error(str.str()); 1940 } 1941 } 1942 1943 filesToGroup.clear(); 1944 delete groupFile; 1945 } 1946 #endif //HAVE_FITS 1985 1947 // -------------------------------------------------------------------------- 1986 1948 // … … 1991 1953 int DataLogger::StopRunPlease() 1992 1954 { 1955 1993 1956 if (fDebugIsOn) 1994 1957 { … … 2020 1983 if (fNumSubAndFitsIsOn) 2021 1984 fNumSubAndFits->updateService(); 1985 1986 CreateFitsGrouping(true); 1987 2022 1988 return kSM_WaitingRun; 2023 1989 … … 2073 2039 fNumSubAndFits->updateService(); 2074 2040 } 2041 CreateFitsGrouping(true); 2042 CreateFitsGrouping(false); 2043 2075 2044 return kSM_Ready; 2076 2045 } … … 2095 2064 2096 2065 //Set the block or allow list 2097 fGreyList.clear(); 2066 fBlackList.clear(); 2067 fWhiteList.clear(); 2098 2068 if (conf.Has("block")) 2099 2069 { … … 2101 2071 if (vec.size() != 0) 2102 2072 { 2103 f IsBlackList = true;2073 fHasBlackList = true; 2104 2074 if (fDebugIsOn) 2105 2075 Debug("Setting BLOCK list:"); 2106 2076 } 2107 else if (conf.Has("allow"))2077 for (vector<string>::iterator it = vec.begin(); it != vec.end(); it++) 2108 2078 { 2109 vec = conf.Get<vector<string>>("allow"); 2110 if (vec.size() != 0) 2111 { 2112 fIsBlackList = false; 2113 if (fDebugIsOn) 2114 Debug("Setting ALLOW list:"); 2115 } 2079 fBlackList.insert(*it); 2080 if (fDebugIsOn) 2081 Debug(" " + *it); 2082 } 2083 } 2084 if (conf.Has("allow")) 2085 { 2086 vector<string> vec = conf.Get<vector<string>>("allow"); 2087 if (vec.size() != 0) 2088 { 2089 fHasWhiteList = true; 2090 if (fDebugIsOn) 2091 Debug("Setting ALLOW list:"); 2116 2092 } 2117 2093 for (vector<string>::iterator it=vec.begin(); it != vec.end(); it++) 2118 2094 { 2119 f GreyList.insert(*it);2095 fWhiteList.insert(*it); 2120 2096 if (fDebugIsOn) 2121 2097 Debug(" " + *it); 2122 2098 } 2123 2099 } 2124 2125 2100 return true; 2126 2101 } 2127 /* 2128 void DataLogger::setBlackWhiteList(const std::string& black, bool isBlack) 2129 { 2130 if (fDebugIsOn) 2131 { 2132 if (isBlack) 2133 Debug("Setting BLACK list: " + black); 2134 else 2135 Debug("Setting WHITE list: " + black); 2136 } 2137 fGreyList.clear(); 2138 stringstream stream(black); 2139 2140 string buffer; 2141 while (getline(stream, buffer, '|')) { 2142 fGreyList.insert(buffer); 2143 } 2144 fIsBlackList = isBlack; 2145 } 2146 */ 2102 2147 2103 // -------------------------------------------------------------------------- 2148 2104
Note:
See TracChangeset
for help on using the changeset viewer.