Changeset 10364

Timestamp:

04/13/11 11:06:40 (14 years ago)

Author:

lyard

Message:

Added FactFits class

Location:

trunk/FACT++/src

Files:

• Converter.cc (modified) (1 diff)
• Converter.h (modified) (1 diff)
• dataLogger.cc (modified) (30 diffs)

trunk/FACT++/src/Converter.cc

@@ -849 +849 @@
 }
 
-void Converter::ToFits(void* dest, const void* src, size_t size)
-{
-        //crawl through the src buffer and copy the data appropriately to the destination buffer
-        //Assumption: the string is always last. This way we use the provided size to determine the number
-        //of character to copy
-        char * charDest = static_cast<char*>(dest);
-        const char* charSrc = static_cast<const char*>(src);
-        size_t cOffset = 0;
-        int elemSize = 0;
-
-        for (unsigned int i=0;i<fList.size()-1;i++)
-        {
-                if (fList[i].first.first->name()[0] == 'S') //string
-                {
-                        for (size_t j=cOffset;j<size;j++)
-                                charDest[j] = charSrc[j];                       
-                        continue;
-                }
+
+void Converter::ToFits(void *dest, const void *src, size_t size) const
+{
+   // crawl through the src buffer and copy the data appropriately to the
+   // destination buffer
+   // Assumption: the string is always last. This way we
+   // use the provided size to determine the number
+   // of character to copy
+
+   char       *charDest = static_cast<char*>(dest);
+   const char *charSrc  = static_cast<const char*>(src);
+
+   for (Converter::FormatList::const_iterator i=fList.begin(); i!=fList.end(); i++)
+   {
+//ETIENNE this check fails for very fine cases. Disabled it
+       // FIXME: This is still not really safe
+//       if (charDest-size>=dest)
+//       {
+//           ostringstream err;
+//           err << "Format description [fmt=" << fFormat << "] exceeds available data size (" << size << ")";
+//           throw runtime_error(err.str());
+//       }
+
+       const char type = i->first.first->name()[0];
+
+       // string types
+       if (type=='S' || type=='O' || type=='W')
+       {
+           // copy string until termination
+           while (*charSrc)
+               *charDest++ = *charSrc++;
+
+           // Copy \0-termination
+//ETIENNE: Do not copy the \0 as it must not be written to fits files. just increment charSrc instead
+ //          *charDest++ = *charSrc++;
+                        charSrc++;
+           continue;
+       }
+
+       const int s = i->first.second;      // size of element
+       const int n = i->second.first;      // number of elements
+
+       // for all elements of this column
+       for (int j=0; j<n; j++)
+       {
+ //ETIENNE moved the +s-1 to the second argument and removed the -1
+           reverse_copy(charSrc,  charSrc+s, charDest);
                 
-                elemSize = fList[i].first.second;
-                for (int j=0;j<fList[i].second.first;j++)//for all elements of this column
-                {
-                        for (int k=0;k<elemSize;k++)//swap the element bytes
-                                charDest[cOffset+k] = charSrc[cOffset + elemSize - (k+1)];
-                        cOffset += elemSize;
-                }
-        }
-}
-
-
-
-
-
-
-
-
+           charSrc  += s;
+           charDest += s;
+       }
+   }
+
+   if (charDest-size!=dest)
+   {
+       ostringstream err;
+       err << "Data block size (" << size << ") doesn't fit format description [fmt=" << fFormat << "]";
+       throw runtime_error(err.str());
+   }
+}
+
+vector<char> Converter::ToFits(const void *src, size_t size) const
+{
+   vector<char> dest(size);
+   ToFits(&dest[0], src, size);
+   return dest;
+}
+
+
+
+
+
+
+
+

trunk/FACT++/src/Converter.h

@@ -91 +91 @@
     static std::string GetHex(const void *d, size_t size, size_t chunk=1);
     
-    void ToFits(void* dest, const void* src, size_t size);
+    void ToFits(void* dest, const void* src, size_t size) const;
+    
+    std::vector<char> ToFits(const void* src, size_t size) const; 
 
 /*

trunk/FACT++/src/dataLogger.cc

@@ -52 +52 @@
 #include "LocalControl.h"
 
-//#define HAS_FITS
+#include "Description.h"
+
+#define HAS_FITS
 
 #include <fstream>
@@ -59 +61 @@
 
 #ifdef HAS_FITS
-#include <astroroot.h>
+//#include <astroroot.h>
+#include "FactFits.h"
 #endif
 
@@ -96 +99 @@
         int fRunNumber; 
         ///Current year
-        short fYear;
+//      short fYear;
         ///Current Month
-        short fMonth;
+//      short fMonth;
         ///Current Day
-        short fDay;
+//      short fDay;
         ///Current Hour
-        short fHour;
+//      short fHour;
         ///Current Minute
-        short fMin;
+//      short fMin;
         ///Current Second
-        short fSec;
+//      short fSec;
         ///Current Milliseconds
-        int fMs;
+//      int fMs;
         ///Current Service Quality
         int fQuality;
@@ -152 +155 @@
 #ifdef HAS_FITS
                 ///daily FITS output file
-                AstroRootIo     dailyFile;
+                FactFits        dailyFile;
                 ///run-specific FITS output file
-                AstroRootIo     runFile;
+                FactFits        runFile;
 #endif
                 ///the actual dimInfo pointer
                 DimStampedInfo* dimInfo;
+                ///the converter for outputting the data according to the format
+                Converter* fConv;
                 ///the number of existing handlers to this structure.
                 ///This is required otherwise I MUST handle the deleting of dimInfo outside from the destructor
@@ -169 +174 @@
                         dimInfo = other.dimInfo;        
                         numCopies = other.numCopies;
+                        fConv = other.fConv;
                         (*numCopies)++;
                 }
@@ -179 +185 @@
                         dimInfo = other.dimInfo;
                         numCopies = other.numCopies;
+                        fConv = other.fConv;
                         (*numCopies)++; 
                 }
@@ -184 +191 @@
                 {
                         dimInfo = info; 
+                        fConv = NULL;
                         numCopies = new int(1);
                 }
@@ -189 +197 @@
                 {
                         dimInfo = NULL;
+                        fConv = NULL;
                         numCopies = new int(1);
                 }
@@ -208 +217 @@
                                 if (numCopies)  
                                 delete numCopies;
+                                delete fConv;
+                                fConv = NULL;
                                 dimInfo = NULL;
                                 numCopies = NULL;
@@ -231 +242 @@
         ///checks whether or not the current info being treated is a run number
         void CheckForRunNumber(DimInfo* I); 
-
         /// start transition
         int StartPlease(); 
@@ -278 +288 @@
 //
 DataLogger::DataLogger(std::ostream &out) : StateMachineDim(out, "DATA_LOGGER")
-{
+{       
                 //initialize member data
                 fDailyFileName = "/home/lyard/log";
@@ -290 +300 @@
                 AddStateName(kSM_BadRunConfig,   "ErrRunFolder");
 
-                /*Add the possible transitions for this machine*/
-
-                //start the daily logging. daily file location must be specified already
+        /*Add the possible transitions for this machine*/
+
+        //start the daily logging. daily file location must be specified already
                 AddTransition(kSM_DailyOpen, fTransStart, kSM_Ready, kSM_BadDailyConfig). 
                     AssignFunction(boost::bind(&DataLogger::StartPlease, this));
 
-                //stop the data logging
+        //stop the data logging
                 AddTransition(kSM_Ready, fTransStop, kSM_DailyOpen, kSM_WaitingRun, kSM_Logging). 
                     AssignFunction(boost::bind(&DataLogger::GoToReadyPlease, this));
 
-                //start the run logging. run file location must be specified already.
-                AddTransition(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig).
+        //start the run logging. run file location must be specified already.
+        AddTransition(kSM_Logging, fTransStartRun, kSM_WaitingRun, kSM_BadRunConfig).
                     AssignFunction(boost::bind(&DataLogger::StartRunPlease, this));
 
-                AddTransition(kSM_WaitingRun, fTransStopRun, kSM_Logging).
+        AddTransition(kSM_WaitingRun, fTransStopRun, kSM_Logging).
                     AssignFunction(boost::bind(&DataLogger::StopRunPlease, this));
 
-                 //transition to exit error states. dunno if required or not, would close the daily file if already openned.
+        //transition to exit error states. dunno if required or not, would close the daily file if already openned.
                 AddTransition(kSM_Ready, fTransReset, kSM_Error, kSM_BadDailyConfig, kSM_BadRunConfig, kSM_Error).
                     AssignFunction(boost::bind(&DataLogger::GoToReadyPlease, this));
 
-                AddTransition(kSM_WaitingRun, fTransWait, kSM_DailyOpen).
+        AddTransition(kSM_WaitingRun, fTransWait, kSM_DailyOpen).
                     AssignFunction(boost::bind(&DataLogger::DailyToWaitRunPlease, this));
 
-                /*Add the possible configurations for this machine*/
-                //configure the daily file location. cannot be done before the file is actually opened
+        /*Add the possible configurations for this machine*/
+        
+        //configure the daily file location. cannot be done before the file is actually opened
                 AddConfiguration(fConfigDay, "C", kSM_Ready, kSM_BadDailyConfig).
                     AssignFunction(boost::bind(&DataLogger::ConfigureDailyFileName, this, _1));
 
-                 //configure the run file location. cannot be done before the file is actually opened, and not in a dailly related error.
+        //configure the run file location. cannot be done before the file is actually opened, and not in a dailly related error.
                 AddConfiguration(fConfigRun, "C", kSM_Ready, kSM_BadDailyConfig, kSM_DailyOpen, kSM_WaitingRun, kSM_BadRunConfig).
                     AssignFunction(boost::bind(&DataLogger::ConfigureRunFileName, this, _1));
@@ -333 +344 @@
                 fServiceList.SetHandler(this);
                 CheckForServicesUpdate();
+                
 }
 // --------------------------------------------------------------------------
@@ -341 +353 @@
 //!     add the configuration (using the conf class ?)
 //
+//FIXME The service must be udpated so that I get the first notification. This should not be
 void DataLogger::CheckForServicesUpdate()
 { 
@@ -544 +557 @@
                 case kSM_Logging:
                 case kSM_DailyOpen:
-//TODO check that this is indeed correct
-                                return 0;//LogMessagePlease(evt);       
+                                return 0;
                 break;
 
         }       
-        //Getting here means that an invalid configuration has been asked.
-        //TODO Log an error message
-        //and return the fatal error state
+
         return kSM_FatalError;
 }
@@ -572 +582 @@
         std::map<std::string, SubscriptionType>::iterator y;
         for (x=fServiceSubscriptions.begin(); x != fServiceSubscriptions.end(); x++)
-        {//instead of extracting the server and service names, I crawl through my records. dunno what's faster, but both should work
+        {//find current service is subscriptions
                 for (y=x->second.begin(); y!=x->second.end();y++)
                         if (y->second.dimInfo == I)
@@ -617 +627 @@
 //!     The current DimInfo 
 //
-void DataLogger::ReportPlease(DimInfo* I, SubscriptionType&)
+void DataLogger::ReportPlease(DimInfo* I, SubscriptionType& sub)
 {
         //should we log or report this info ? (i.e. is it a message ?)
         bool isItaReport = ((strstr(I->getName(), "Message") == NULL) && (strstr(I->getName(), "MESSAGE") == NULL));
         
-//      std::ofstream & dailyFile = fDailyReportFile;//isItaReport? fDailyReportFile : fDailyLogFile;
-//      std::ofstream & runFile = fRunReportFile;//isItaReport? fRunReportFile : fRunLogFile;
-
         //TODO add service exclusion
+        
         if (!fDailyReportFile.is_open())
                 return;
+
+        //create the converter for that service
+        if (sub.fConv == NULL)
+        {
+                sub.fConv = new Converter(Out(), I->getFormat());       
+                if (!sub.fConv)
+                {
+                        Error("Couldn't properly parse the format... service ignored.");
+                        return; 
+                }
+        }
                 
         //construct the header
         std::stringstream header;
-        
         Time cTime((time_t)(I->getTimestamp()), I->getTimestampMillisecs());
-
-        //Buffer them for FITS write
-        //TODO this has been replaced by MjD. So I guess that these member variables can go.
-        fYear = cTime.Y(); fMonth = cTime.M(); fDay = cTime.D();
-        fHour = cTime.h(); fMin = cTime.m(); fSec = cTime.s();
-        fMs = cTime.ms(); fQuality = I->getQuality();
-        
+        fQuality = I->getQuality();
         fMjD = cTime.Mjd();
-        
+
         if (isItaReport)
         {
                 //write text header
                 header << I->getName() << " " << fQuality << " ";
-                header << fYear << " " << fMonth << " " << fDay << " ";
-                header << fHour << " " << fMin << " " << fSec << " ";
-                header << fMs << " " << I->getTimestamp() << " ";
-
-                const Converter conv(Out(), I->getFormat());
-                if (!conv)
-                {
-                    Error("Couldn't properly parse the format... ignored.");
-                    return;
-                }
-
-                std::string text;
-                try
-                {
-                    text = conv.GetString(I->getData(), I->getSize());
-                }
-                catch (const std::runtime_error &e)
-                {
-                    Out() << kRed << e.what() << endl;
-                    Error("Couldn't properly parse the data... ignored.");
-                    return;
-                }
+                header << cTime.Y() << " " << cTime.M() << " " << cTime.D() << " ";
+                header << cTime.h() << " " << cTime.m() << " " << cTime.s() << " ";
+                header << cTime.ms() << " " << I->getTimestamp() << " ";
+
+                std::string text;
+        try
+        {
+                text = sub.fConv->GetString(I->getData(), I->getSize());
+        }
+        catch (const std::runtime_error &e)
+        {
+                Out() << kRed << e.what() << endl;
+            Error("Couldn't properly parse the data... ignored.");
+            return;
+        }
 
                 if (text.empty())
-                    return;
-
-                //replace bizarre characters by white space
-                replace(text.begin(), text.end(), '\n', '\\');
-                replace_if(text.begin(), text.end(), std::ptr_fun<int, int>(&std::iscntrl), ' ');
+                return;
+
+        //replace bizarre characters by white space
+        replace(text.begin(), text.end(), '\n', '\\');
+        replace_if(text.begin(), text.end(), std::ptr_fun<int, int>(&std::iscntrl), ' ');
         
                 if (fDailyReportFile.is_open())
@@ -681 +686 @@
                         fRunReportFile << header.str();
 
-                if (fDailyReportFile.is_open())
+        if (fDailyReportFile.is_open())
                         fDailyReportFile << text << std::endl;
                 if (fRunReportFile.is_open())
@@ -699 +704 @@
                 }
         }
-        
+
 #ifdef HAS_FITS
         if (!sub.dailyFile.IsOpen() || !sub.runFile.IsOpen())
                 OpenFITSFilesPlease(sub);       
         WriteToFITS(sub);
+                
 #endif
 
@@ -718 +724 @@
 //!    I guess that this function should not be any longer
 //
+//TODO isn't that function not used any longer ? If so I guess that we should get rid of it...
 int DataLogger::LogMessagePlease(const Event& evt)
 {
@@ -729 +736 @@
         header << cTime.ms() << " ";
                 
-//      std::string text = ToString(evt.GetFormat().c_str(), evt.GetData(), evt.GetSize());
-
-        const Converter conv(Out(), evt.GetFormat());
-        if (!conv)
-        {
-            Error("Couldn't properly parse the format... ignored.");
-            return GetCurrentState();
-        }
-
-        std::string text;
-        try
-        {
-            text = conv.GetString(evt.GetData(), evt.GetSize());
-        }
-        catch (const std::runtime_error &e)
-        {
-            Out() << kRed << e.what() << endl;
-            Error("Couldn't properly parse the data... ignored.");
-            return GetCurrentState();
-        }
-
-
-        if (text.empty())
-            return GetCurrentState();
-
-        //replace bizarre characters by white space
-        replace(text.begin(), text.end(), '\n', '\\');
-        replace_if(text.begin(), text.end(), std::ptr_fun<int, int>(&std::iscntrl), ' ');
+    const Converter conv(Out(), evt.GetFormat());
+    if (!conv)
+    {
+        Error("Couldn't properly parse the format... ignored.");
+        return GetCurrentState();
+    }
+
+    std::string text;
+    try
+    {
+        text = conv.GetString(evt.GetData(), evt.GetSize());
+    }
+    catch (const std::runtime_error &e)
+    {
+        Out() << kRed << e.what() << endl;
+        Error("Couldn't properly parse the data... ignored.");
+        return GetCurrentState();
+    }
+
+        if (text.empty())
+        return GetCurrentState();
+
+    //replace bizarre characters by white space
+    replace(text.begin(), text.end(), '\n', '\\');
+    replace_if(text.begin(), text.end(), std::ptr_fun<int, int>(&std::iscntrl), ' ');
 
         if (fDailyLogFile.is_open())
@@ -834 +838 @@
         fullName = fDailyFileName + '/' + sTime.str() + ".rep";
         fDailyReportFile.open(fullName.c_str(), std::ios_base::out | std::ios_base::app);
+        
         if (!fDailyLogFile.is_open() || !fDailyReportFile.is_open())
-        {
+        {       
                 //TODO send an error message    
             return kSM_BadDailyConfig;
         }
+
         return kSM_DailyOpen;   
 }
@@ -866 +872 @@
         {
                 std::string partialName = fDailyFileName + '/' + sTime.str() + '_' + serviceName + ".fits";
-                std::string fullName = fDailyFileName + '/' + sTime.str() + '_' + serviceName + ".fits[" + serviceName + "]";
-
                 AllocateFITSBuffers(sub);
-                //currently, the FITS are written in the same directory as the text files. 
-                //thus the write permissions have already been checked by the text files. 
-                //if the target folder changes, then I should check the write permissions here
-                //now we only check whether the target file exists or not
-                std::ifstream readTest(partialName.c_str());
-                if (readTest.is_open())
-                {
-                        readTest.close();
-                        sub.dailyFile.Open(fullName.c_str(), "UPDATE");
-                }
-                else {
-                        sub.dailyFile.Open(fullName.c_str(), "CREATE");  
-                }
-                
-//TODO Write the header's attributes
+                sub.dailyFile.Open(partialName, serviceName);
         }
         if (!sub.runFile.IsOpen() && (GetCurrentState() == kSM_Logging))
@@ -890 +880 @@
                 sRun << fRunNumber;
                 std::string partialName = fRunFileName + '/' + sRun.str() + '_' + serviceName + ".fits";
-                std::string fullName = fRunFileName + '/' +  sRun.str() + '_' + serviceName + ".fits[" + serviceName + "]";
-                
-                std::ifstream readTest(partialName.c_str());
-                if (readTest.is_open())
-                {
-                        readTest.close();
-                        sub.runFile.Open(fullName.c_str(), "UPDATE");
-                }
-                else
-                        sub.runFile.Open(fullName.c_str(), "CREATE");
-//TODO Write the header's attributes
+                sub.runFile.Open(partialName, serviceName);
         }
 }       
@@ -907 +887 @@
 void DataLogger::AllocateFITSBuffers(SubscriptionType& sub)
 {
-         const char* format = sub.dimInfo->getFormat();
-         const int size = sub.dimInfo->getSize();
+        int size = sub.dimInfo->getSize();
          
-         //Init the time columns of the file
-         sub.dailyFile.InitCol("Date", "double", &fMjD);
-         sub.runFile.InitCol("Date", "double", &fMjD);
-         
-//       sub.dailyFile.InitCol("Year", "short", &fYear);
-//       sub.dailyFile.InitCol("Month", "short", &fMonth);
-//       sub.dailyFile.InitCol("Day", "short", &fDay);
-//       sub.dailyFile.InitCol("Hour", "short", &fHour);
-//       sub.dailyFile.InitCol("Minute", "short", &fMin);
-//       sub.dailyFile.InitCol("Second", "short", &fSec);
-//       sub.dailyFile.InitCol("MilliSec", "int", &fMs);
-         sub.dailyFile.InitCol("QoS", "int", &fQuality);
-
-//       sub.runFile.InitCol("Year", "short", &fYear);
-//       sub.runFile.InitCol("Month", "short", &fMonth);
-//       sub.runFile.InitCol("Day", "short", &fDay);
-//       sub.runFile.InitCol("Hour", "short", &fHour);
-//       sub.runFile.InitCol("Minute", "short", &fMin);
-//       sub.runFile.InitCol("Second", "short", &fSec);
-//       sub.runFile.InitCol("MilliSec", "int", &fMs);
-         sub.runFile.InitCol("QoS", "int", &fQuality);
-
-         const Converter::FormatList flist = Converter::Compile(Out(), format);
-
-         // Compilation failed
-         if (fList.empty() || fList.back().first.second!=0)
-         {
-             Error("Compilation of format string failed.");
-             return;
-         }
-
-         //we've got a nice structure describing the format of this service's messages.
-         //Let's create the appropriate FITS columns
-         for (unsigned int i=0;i<flist.size();i++)
-         {
-                std::stringstream colName;
+        //Init the time columns of the file
+        Description dateDesc(std::string("Time"), std::string("MjD"), std::string("Modified Julian Date"));
+        sub.dailyFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
+        sub.runFile.AddStandardColumn(dateDesc, "1D", &fMjD, sizeof(double));
+
+        Description QoSDesc("Qos", "None", "Quality of service");
+        sub.dailyFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
+        sub.runFile.AddStandardColumn(QoSDesc, "1J", &fQuality, sizeof(int));
+
+        const Converter::FormatList flist = sub.fConv->GetList();
+    // Compilation failed
+    if (flist.empty() || flist.back().first.second!=0)
+    {
+        Error("Compilation of format string failed.");
+        return;
+    }
+
+        //we've got a nice structure describing the format of this service's messages.
+        //Let's create the appropriate FITS columns
+        std::vector<std::string> dataFormatsLocal;
+        for (unsigned int i=0;i<flist.size()-1;i++)
+        {
                 std::stringstream dataQualifier; 
-                void * dataPointer = static_cast<char*>(sub.dimInfo->getData()) + flist[i].second.second;
-                colName << "Data" << i;
+
                 dataQualifier << flist[i].second.first;
-                switch (flist[i].first.first)
+                switch (flist[i].first.first->name()[0])
                 {
                         case 'c':
@@ -978 +942 @@
                                 dataQualifier.str(""); //clear
                                 dataQualifier << size-1 <<  "A";
+                                size = size-1;
                         break;
                         
                         default:
-                                Error("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 962.");
+                                Error("THIS SHOULD NEVER BE REACHED. dataLogger.cc ln 948.");
                 };
-                sub.dailyFile.InitCol(colName.str().c_str(), dataQualifier.str().c_str(), dataPointer);
-                sub.runFile.InitCol(colName.str().c_str(), dataQualifier.str().c_str(), dataPointer);
+                dataFormatsLocal.push_back(dataQualifier.str());
          }
 
-//TODO init the attributes
+         sub.dailyFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size);
+         sub.runFile.InitDataColumns(fServiceList.GetDescriptions(sub.dimInfo->getName()), dataFormatsLocal, sub.dimInfo->getData(), size);
 }
 // --------------------------------------------------------------------------
@@ -997 +962 @@
                 //dailyFile status (open or not) already checked
                 if (sub.dailyFile.IsOpen())
-                        sub.dailyFile.Write();
+                        sub.dailyFile.Write(sub.fConv);
                 if (sub.runFile.IsOpen())
-                        sub.runFile.Write();
+                        sub.runFile.Write(sub.fConv);
 }
 #endif //if has_fits
@@ -1131 +1096 @@
 
     logger.SetReady();
+ //  logger.StartPlease();
     shell.Run();                 // Run the shell
-    logger.SetNotReady();
+     logger.SetNotReady();
 
     return 0;
@@ -1183 +1149 @@
 
     // To allow overwriting of DIM_DNS_NODE set 0 to 1
-    setenv("DIM_DNS_NODE", conf.Get<std::string>("dns").c_str(), 1);
+ //   setenv("DIM_DNS_NODE", conf.Get<std::string>("dns").c_str(), 0);
 
     try
@@ -1191 +1157 @@
             return RunDim(conf);
 
-        // Cosole access w/ and w/o Dim
+        // Console access w/ and w/o Dim
         if (conf.Get<int>("console")==0)
             return RunShell<LocalShell>(conf);