Changeset 10739

Timestamp:

05/18/11 12:01:15 (14 years ago)

Author:

lyard

Message:

Added Fits grouping

File:

• trunk/FACT++/src/dataLogger.cc (modified) (25 diffs)

trunk/FACT++/src/dataLogger.cc

@@ -66 +66 @@
 #include <sys/stat.h>
 
-//#define HAVE_FITS
-//#define ONE_RUN_FITS_ONLY
-
 #include <fstream>
 
@@ -110 +107 @@
         void setQuality(int){};
 };
-class DataLogger : public StateMachineDim, DimInfoHandler//, DimServiceInfoList //,DimInfoHandler 
+class DataLogger : public StateMachineDim, DimInfoHandler
 {
 public:
@@ -287 +284 @@
 #ifdef ONE_RUN_FITS_ONLY
         ///FITS file for runs. only one, hence dealt with in the dataLogger itself
-        FITS* fRunFitsFile;
+        CCfits::FITS* fRunFitsFile;
 #endif //one_run_fits_only
 #endif//has_fits
@@ -317 +314 @@
 
         inline void NotifyOpenedFile(std::string name, int type, DimDescribedService* service);
+
 public: 
-//      void setBlackWhiteList(const std::string& , bool);
         bool SetConfiguration(Configuration& conf);
 
 private:
-        std::set<std::string> fGreyList;
-        bool fIsBlackList;
+        std::set<std::string> fBlackList;
+        std::set<std::string> fWhiteList;
+        bool fHasBlackList;
+        bool fHasWhiteList;
         bool fDebugIsOn;
         float fStatsPeriodDuration;
@@ -338 +337 @@
         ///Small function for calculating the total size written so far
         void calculateTotalSizeWritten(DataLoggerStats& statVar, bool& shouldWarn, bool isPrinting);
+
+        ///vectors to keep track of opened Fits files, for grouping purposes.
+        //This cannot be done otherwise, as services may disapear before the files are nicely closed. Hence which files were opened must be remembered.
+        map<string, string> fOpenedRunFits;
+        map<string, string> fOpenedNightlyFits;
+        void CreateFitsGrouping(bool runGroup);
 }; //DataLogger
 
@@ -444 +449 @@
 void DataLogger::ServicesMonitoring()
 {
-                //create the DIM service
-//              int dataSize = 2*sizeof(long) + sizeof(long);
-
                 DataLoggerStats statVar;
                 statVar.sizeWritten = 0;
@@ -472 +474 @@
                                 sleep(fStatsPeriodDuration);
                         //update the fits files sizes
-                        calculateTotalSizeWritten(statVar, statWarning, false);/*
-#ifdef HAVE_FITS
-                        SubscriptionsListType::iterator x;
-                        std::map<std::string, SubscriptionType>::iterator y;
-                        bool runFileDone = false;
-                        for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
-                        {
-                                for (y=x->second.begin(); y != x->second.end(); y++)
-                                {
-                                        if (y->second.runFile.IsOpen() && !runFileDone)
-                                        {
-                                                        fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();
-#ifdef ONE_FITS_ONLY
-                                                        runFileDone = true;
-#endif
-                                        }
-                                        if (y->second.nightlyFile.IsOpen())
-                                                fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();
-                                }
-                        }
-#endif
-                        struct stat st;
-                        //gather log and report files sizes on disk
-                        if (fNightlyLogFile.is_open())
-                        {
-                                stat(fFullNightlyLogFileName.c_str(), &st);
-                                fFileSizesMap[fFullNightlyLogFileName] = st.st_size;    
-                        }
-                        if (fNightlyReportFile.is_open())
-                        {
-                                stat(fFullNightlyReportFileName.c_str(), &st);
-                                fFileSizesMap[fFullNightlyReportFileName] = st.st_size; 
-                        }
-                        if (fRunLogFile.is_open())
-                        {
-                                stat(fFullRunLogFileName.c_str(), &st);
-                                fFileSizesMap[fFullRunLogFileName] = st.st_size;        
-                        }
-                        if (fRunReportFile.is_open())
-                        {
-                                stat(fFullRunReportFileName.c_str(), &st);
-                                fFileSizesMap[fFullRunReportFileName] = st.st_size;
-                        }       
-
-                        if (!statvfs(fNightlyFileName.c_str(), &vfs))
-                        {
-                                statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
-                                statWarning = false;
-                        }
-                        else
-                        {
-                                std::stringstream str;
-                                str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
-                                if (!statWarning)
-                                        Error(str);
-                                statWarning = true;
-                                statVar.freeSpace = -1;
-                        }
-                        
-                        //sum up all the file sizes. past and present
-                        statVar.sizeWritten = 0;
-                        for (std::map<std::string, long>::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end();  it++)
-                                statVar.sizeWritten += it->second;
-                        statVar.sizeWritten -= fBaseSizeNightly;
-                        statVar.sizeWritten -= fBaseSizeRun;
-                        */
+                        calculateTotalSizeWritten(statVar, statWarning, false);
                         if (fStatsPeriodDuration == 0.0f)
                                 continue;
@@ -570 +507 @@
     dic_disable_padding();
     dis_disable_padding();
-
                 //initialize member data
-                fNightlyFileName = ".";//"/home/lyard/log";//
-                fRunFileName = ".";//"/home/lyard/log";
+                fNightlyFileName = ".";
+                fRunFileName = ".";
                 fRunNumber = -1;
                 fPreviousRunNumber = fRunNumber;
@@ -677 +613 @@
                                                          "|NumOpenFiles[int]:number of files currently open by the data logger");
 
-                //black/white list
-                fIsBlackList = true;
-                fGreyList.clear();
-        
+        //black/white list
+        fHasBlackList = false;
+        fHasWhiteList = false;
+                fBlackList.clear();
+                fWhiteList.clear();
                 //services parameters
                 fDebugIsOn = false;
@@ -750 +687 @@
                 if ((i->find("DIS_DNS") != std::string::npos) ||
                     (i->find("DATA_LOGGER") != std::string::npos))
-                        continue;
-                if (fIsBlackList && (fGreyList.find(*i) != fGreyList.end()))
                         continue;
                 //find the current server in our subscription list      
@@ -781 +716 @@
                                 if (*givenSubs == "SERVICE_LIST")
                                         continue;
-                                if (fIsBlackList && fGreyList.find(*givenSubs) != fGreyList.end())
-                                        continue;
-
-                                if (fIsBlackList && fGreyList.find((*i) + "/" + (*givenSubs)) != fGreyList.end())
-                                        continue;
-                                else if (!fIsBlackList && 
-                                        (fGreyList.find((*i) + "/" + (*givenSubs)) == fGreyList.end()) &&
-                                        (fGreyList.find(*i) == fGreyList.end()) &&
-                                        (fGreyList.find(*givenSubs) == fGreyList.end()))
-                                                continue;
-                                if (cSubs->second.find(*givenSubs) == cSubs->second.end())
+
+                                if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) &&
+                                     (fWhiteList.find(*i + "/" + *givenSubs) == fWhiteList.end()) &&
+                                     (fWhiteList.find("/" + *givenSubs) == fWhiteList.end()))
+                                    continue;
+                if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) ||
+                                      (fBlackList.find(*i + "/" + *givenSubs) != fBlackList.end()) ||
+                                      (fBlackList.find("/" + *givenSubs) != fBlackList.end())))
+                    continue;
+
+                if (cSubs->second.find(*givenSubs) == cSubs->second.end())
                                 {//service not found. Add it
                                         cSubs->second[*givenSubs].dimInfo = new DimStampedInfo(((*i) + "/" + *givenSubs).c_str(), const_cast<char*>(""), this);
@@ -812 +747 @@
                                 if (*j == "SERVICE_LIST")
                                         continue;
-                                if (fIsBlackList && fGreyList.find(*j) != fGreyList.end())
-                                        continue;
-
-                                if (fIsBlackList && fGreyList.find((*i) + "/" + (*j)) != fGreyList.end())
-                                        continue;
-                                else if (!fIsBlackList && 
-                                        (fGreyList.find((*i) + "/" + (*j)) == fGreyList.end()) &&
-                                        (fGreyList.find(*i) == fGreyList.end()) &&
-                                        (fGreyList.find(*j) == fGreyList.end()))
-                                                continue;
+                if (fHasWhiteList && (fWhiteList.find(*i + "/") == fWhiteList.end()) &&
+                                     (fWhiteList.find(*i + "/" + *j) == fWhiteList.end()) &&
+                                     (fWhiteList.find("/" + *j) == fWhiteList.end()))
+                    continue;
+                if (fHasBlackList && ((fBlackList.find(*i + "/") != fBlackList.end()) ||
+                                      (fBlackList.find(*i + "/" + *j) != fBlackList.end()) ||
+                                      (fBlackList.find("/" + *j) != fBlackList.end())))
+                    continue;
+
                                         
                                 liste[*j].dimInfo = new DimStampedInfo(((*i) + "/" + (*j)).c_str(), const_cast<char*>(""), this);
@@ -1294 +1228 @@
         bool statWarning = false;
         DataLoggerStats statVar;
-        calculateTotalSizeWritten(statVar, statWarning, false);/*
-#ifdef HAVE_FITS
-        str.str("");
-        str << "There are " << fNumSubAndFitsData.numOpenFits << " FITS files open:";
-        Message(str.str());
-        SubscriptionsListType::iterator x;
-        std::map<std::string, SubscriptionType>::iterator y;
-        bool runFileDone = false;
-        for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
-        {
-                for (y=x->second.begin(); y != x->second.end(); y++)
-                {
-                        if (y->second.runFile.IsOpen() && !runFileDone)
-                        {
-                                        fFileSizesMap[y->second.runFile.fFileName] = y->second.runFile.GetWrittenSize();
-                                        Message("-> "+y->second.runFile.fFileName);
-#ifdef ONE_FITS_ONLY
-                                        runFileDone = true;
-#endif
-                        }
-                        if (y->second.nightlyFile.IsOpen())
-                        {
-                                fFileSizesMap[y->second.nightlyFile.fFileName] = y->second.nightlyFile.GetWrittenSize();
-                                Message("-> "+y->second.nightlyFile.fFileName);
-                        }
-                }
-        }
-#else
-        Message("FITS output disabled at compilation");
-#endif
-        struct stat st;
-        DataLoggerStats statVar;
-        //gather log and report files sizes on disk
-        if (fNightlyLogFile.is_open())
-        {
-                stat(fFullNightlyLogFileName.c_str(), &st);
-                fFileSizesMap[fFullNightlyLogFileName] = st.st_size;    
-        }
-        if (fNightlyReportFile.is_open())
-        {
-                stat(fFullNightlyReportFileName.c_str(), &st);
-                fFileSizesMap[fFullNightlyReportFileName] = st.st_size; 
-        }
-        if (fRunLogFile.is_open())
-        {
-                stat(fFullRunLogFileName.c_str(), &st);
-                fFileSizesMap[fFullRunLogFileName] = st.st_size;        
-        }
-        if (fRunReportFile.is_open())
-        {
-                stat(fFullRunReportFileName.c_str(), &st);
-                fFileSizesMap[fFullRunReportFileName] = st.st_size;
-        }       
-        struct statvfs vfs;
-        if (!statvfs(fNightlyFileName.c_str(), &vfs))
-        {
-                statVar.freeSpace = vfs.f_bsize*vfs.f_bavail;
-        }
-        else
-        {
-                str.str("");
-                str << "Unable to retrieve stats for " << fNightlyFileName << ". Reason: " << strerror(errno) << " [" << errno << "]";
-                Error(str);;
-                statVar.freeSpace = -1;
-        }
-        
-        //sum up all the file sizes. past and present
-        statVar.sizeWritten = 0;
-        for (std::map<std::string, long>::iterator it=fFileSizesMap.begin(); it != fFileSizesMap.end();  it++)
-                statVar.sizeWritten += it->second;
-        statVar.sizeWritten -= fBaseSizeNightly;
-        statVar.sizeWritten -= fBaseSizeRun;
-        */
+        calculateTotalSizeWritten(statVar, statWarning, false);
         Message("---------------- STATS ------------------");
         str.str("");
@@ -1392 +1254 @@
                         Message(" -> "+it2->first);
         }
-        if (fIsBlackList)
-                Message("------------- BLOCK LIST ----------------");
-        else
-                if (fGreyList.size() != 0)
+        if (fHasBlackList)
+        {
+            Message("------------- BLOCK LIST ----------------");
+            for (set<string>::iterator it=fBlackList.begin(); it != fBlackList.end(); it++)
+                Message(*it);
+        }
+        if (fHasWhiteList)
+        {
                         Message("----------- ALLOW LIST ------------------");
-        for (set<string>::iterator it=fGreyList.begin(); it != fGreyList.end(); it++)
-            Message(*it);
-
-        if (fGreyList.size()==0)
-            Message(" <empty>");
+                        for (set<string>::iterator it=fWhiteList.begin(); it != fWhiteList.end(); it++)
+                            Message(*it);
+        }
 
         return GetCurrentState();
@@ -1665 +1529 @@
         //notify that a new file has been opened.
         NotifyOpenedFile(actualTargetDir + '/' + sTime.str(), 3, fOpenedNightlyFiles);
+
+        fOpenedNightlyFits.clear();
         
         return kSM_NightlyOpen;         
@@ -1692 +1558 @@
         if (!sub.nightlyFile.IsOpen())
         {
-                std::string partialName = fNightlyFileName + '/' + sTime.str() + '_' + serviceName + ".fits";
+            std::string fileNameOnly = sTime.str() + '_' + serviceName + ".fits";
+                std::string partialName = fNightlyFileName + '/' + fileNameOnly;
                 AllocateFITSBuffers(sub);
                 //get the size of the file we're about to open
@@ -1701 +1568 @@
                                 fBaseSizeNightly += st.st_size;
                         fFileSizesMap[partialName] = 0;
+                        //remember that this file was opened.
+                        fOpenedNightlyFits[fileNameOnly] =  serviceName;
                 }
                 sub.nightlyFile.Open(partialName, serviceName, NULL, &fNumSubAndFitsData.numOpenFits, this);//Out());
+
                 //notify the opening
                 std::string actualTargetDir;
@@ -1730 +1600 @@
                 sRun << fRunNumber;
 #ifdef ONE_RUN_FITS_ONLY
-                std::string partialName = fRunFileName + '/' + sRun.str() + ".fits";
+                std::string fileNameOnly = sRun.str() + ".fits";
+                std::string partialName = fRunFileName + '/' + fileNameOnly;
                 if (fRunFitsFile == NULL)
                 {
 #else
-                std::string partialName = fRunFileName + '/' + sRun.str() + '_' + serviceName + ".fits";
+                std::string fileNameOnly = sRun.str() + '_' + serviceName + ".fits";
+                std::string partialName = fRunFileName + '/' + fileNameOnly;
 #endif
                         //get the size of the file we're about to open
@@ -1745 +1617 @@
                                         fBaseSizeRun = 0;
                                 fFileSizesMap[partialName] = 0; 
+                                fOpenedRunFits[fileNameOnly] = serviceName;
                         }
 #ifdef ONE_RUN_FITS_ONLY
                         try
                         {
-                                fRunFitsFile = new FITS(partialName, RWmode::Write);    
+                                fRunFitsFile = new CCfits::FITS(partialName, CCfits::RWmode::Write);
                                 (fNumSubAndFitsData.numOpenFits)++;
                         }       
@@ -1981 +1854 @@
         NotifyOpenedFile(actualTargetDir + '/' + sRun.str(), 3, fOpenedRunFiles);
         
+        fOpenedRunFits.clear();
+
         return kSM_Logging;
 }
+#ifdef HAVE_FITS
+void DataLogger::CreateFitsGrouping(bool runGroup)
+{
+    //create the FITS group corresponding to the ending run.ETIENNEICI
+    CCfits::FITS* groupFile;
+
+#ifdef ONE_RUN_FITS_ONLY
+    if (runGroup)
+        return;
+#endif
+    map<string, string>& filesToGroup = runGroup? fOpenedRunFits : fOpenedNightlyFits;
+    unsigned int numFilesToGroup = filesToGroup.size();
+    if (numFilesToGroup <= 1)
+    {
+        filesToGroup.clear();
+        return;
+    }
+    stringstream groupName;
+    if (runGroup)
+    {
+        groupName << fRunFileName << '/' << fRunNumber << ".fits";
+    }
+    else
+    {
+        Time time;
+        std::stringstream sTime;
+        sTime << time.Y() << "_" << time.M() << "_" << time.D();
+
+        groupName << fNightlyFileName << '/' << sTime.str() << ".fits";
+    }
+    try
+    {
+        groupFile = new CCfits::FITS(groupName.str(), CCfits::RWmode::Write);
+        //setup the column names
+        int maxCharLength = FILENAME_MAX;
+        stringstream pathTypeName;
+        pathTypeName << maxCharLength << "A";
+        vector<string> names;
+        vector<string> dataTypes;
+        names.push_back("MEMBER_XTENSION");
+        dataTypes.push_back("8A");
+        names.push_back("MEMBER_URI_TYPE");
+        dataTypes.push_back("3A");
+        names.push_back("MEMBER_LOCATION");
+        dataTypes.push_back(pathTypeName.str());
+        names.push_back("MEMBER_NAME");
+        dataTypes.push_back(pathTypeName.str());
+        CCfits::Table* groupTable = groupFile->addTable("GROUPING", numFilesToGroup, names, dataTypes);
+     }
+     catch (CCfits::FitsError e)
+     {
+         std::stringstream str;
+         str << "Could not open or create FITS table GROUPING in  file " << groupName.str() << " reason: " << e.message();
+         Error(str);
+         return;
+     }
+
+    //CCfits seems to be buggy somehow: can't use the column's function "write": it create a compilation error: maybe strings were not thought about.
+    //use cfitsio routines instead
+    groupTable->makeThisCurrent();
+    //create appropriate buffer.
+    unsigned char* fitsBuffer = new unsigned char[8 + 3 + 2*maxCharLength + 1]; //+1 for trailling character
+    char* startOfExtension = reinterpret_cast<char*>(fitsBuffer);
+    char* startOfURI = reinterpret_cast<char*>(&fitsBuffer[8]);
+    char* startOfLocation = reinterpret_cast<char*>(&fitsBuffer[8 + 3]);
+    char* startOfName = reinterpret_cast<char*>(&fitsBuffer[8+3+maxCharLength]);
+    sprintf(startOfExtension, "%s", "BINTABLE");
+    sprintf(startOfURI, "%s", "URL");
+    int i=1;
+    for (map<string, string>::iterator it=filesToGroup.begin(); it!=filesToGroup.end(); it++, i++)
+    {
+        strcpy(startOfLocation, it->first.c_str());
+        strcpy(startOfName, it->second.c_str());
+        int status = 0;
+        fits_write_tblbytes(groupFile->fitsPointer(), i, 1, 8+3+2*maxCharLength, fitsBuffer, &status);
+        if (status)
+        {
+            stringstream str;
+            str << "Could not write row #" << i << "In the fits grouping file " << groupName << ". Cfitsio error code: " << status;
+            Error(str.str());
+        }
+    }
+
+    filesToGroup.clear();
+    delete groupFile;
+}
+#endif //HAVE_FITS
 // --------------------------------------------------------------------------
 //
@@ -1991 +1953 @@
 int DataLogger::StopRunPlease()
 {
+
         if (fDebugIsOn)
         {
@@ -2020 +1983 @@
         if (fNumSubAndFitsIsOn)
                 fNumSubAndFits->updateService();
+
+        CreateFitsGrouping(true);
+
         return kSM_WaitingRun;
 
@@ -2073 +2039 @@
                         fNumSubAndFits->updateService();
         }
+        CreateFitsGrouping(true);
+        CreateFitsGrouping(false);
+
         return kSM_Ready;
 }
@@ -2095 +2064 @@
 
         //Set the block or allow list
-        fGreyList.clear();
+        fBlackList.clear();
+        fWhiteList.clear();
     if (conf.Has("block"))
     {
@@ -2101 +2071 @@
         if (vec.size() != 0)
         {
-                fIsBlackList = true;
+                fHasBlackList = true;
                 if (fDebugIsOn)
                                 Debug("Setting BLOCK list:");
         }
-        else if (conf.Has("allow"))
+        for (vector<string>::iterator it = vec.begin(); it != vec.end(); it++)
         {
-                vec = conf.Get<vector<string>>("allow");
-                if (vec.size() != 0)
-                {
-                        fIsBlackList = false;
-                        if (fDebugIsOn)
-                                Debug("Setting ALLOW list:");
-                }
+            fBlackList.insert(*it);
+            if (fDebugIsOn)
+                Debug("                   " + *it);
+        }
+    }
+    if (conf.Has("allow"))
+    {
+        vector<string> vec = conf.Get<vector<string>>("allow");
+        if (vec.size() != 0)
+        {
+                fHasWhiteList = true;
+                if (fDebugIsOn)
+                        Debug("Setting ALLOW list:");
         }
         for (vector<string>::iterator it=vec.begin(); it != vec.end(); it++)
         {
-                fGreyList.insert(*it);
+                fWhiteList.insert(*it);
                 if (fDebugIsOn)
                         Debug("                   " + *it);
         }
     }
-
     return true;
 }
-/*
-void DataLogger::setBlackWhiteList(const std::string& black, bool isBlack)
-{
-        if (fDebugIsOn)
-        {
-                if (isBlack)
-                        Debug("Setting BLACK list: " + black);  
-                else
-                        Debug("Setting WHITE list: " + black);
-        }
-        fGreyList.clear();
-        stringstream stream(black);
-
-        string buffer;
-        while (getline(stream, buffer, '|')) {
-                fGreyList.insert(buffer);       
-        }
-        fIsBlackList = isBlack;
-}
-*/
+
 // --------------------------------------------------------------------------