Changeset 16443 for trunk/Mars/mcore/zfits.h

Timestamp:

05/29/13 22:02:43 (11 years ago)

Author:

lyard

Message:

more tweaks to factfits

File:

• trunk/Mars/mcore/zfits.h (modified) (11 diffs)

trunk/Mars/mcore/zfits.h

@@ -33 +33 @@
         fNumRowsPerTile(0),
         fCurrentRow(-1),
-        fHeapOff(0)
+        fHeapOff(0),
+        fTileSize(0)
     {
         InitCompressionReading();
@@ -47 +48 @@
               fNumRowsPerTile(0),
               fCurrentRow(-1),
-              fHeapOff(0)
+              fHeapOff(0),
+              fTileSize(0)
     {
         InitCompressionReading();
@@ -60 +62 @@
         fRow++;
         return true;
+    }
+protected:
+
+    //  Stage the requested row to internal buffer
+    //  Does NOT return data to users
+    virtual void StageRow(size_t row, char* dest)
+    {
+        if (!fTable.isCompressed)
+        {
+            fits::StageRow(row, dest);
+            return;
+        }
+
+        ReadBinaryRow(row, dest);
     }
 
@@ -82 +98 @@
     }
 
-    //  Stage the requested row to internal buffer
-    //  Does NOT return data to users
-    void StageRow(size_t row, char* dest)
-    {
-        if (!fTable.isCompressed)
-        {
-            fits::StageRow(row, dest);
-            return;
-        }
-
-        ReadBinaryRow(row, dest);
-    }
-
     // Copy decompressed data to location requested by user
     void MoveColumnDataToUserSpace(char* dest, const char* src, const Table::Column& c)
@@ -118 +121 @@
 
     vector<vector<pair<int64_t, int64_t> > > fCatalog;///< Catalog, i.e. the main table that points to the compressed data.
+    vector<size_t> fTileSize; ///< size in bytes of each compressed tile
+    vector<vector<size_t> > fTileOffsets; ///< offset from start of tile of a given compressed column
 
     void AllocateBuffers()
@@ -127 +132 @@
 
         fTransposedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile);
-        fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + 1024*1024); //use a bit more memory, in case the compression algorithms uses more
+        fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + fTable.num_cols); //use a bit more memory for compression flags
     }
 
@@ -139 +144 @@
         fCatalog.resize(fNumTiles);
 
+        const streampos catalogStart = tellg();
+
+        //do the actual reading
         for (uint32_t i=0;i<fNumTiles;i++)
             for (uint32_t j=0;j<fTable.num_cols;j++)
@@ -147 +155 @@
                 int64_t tempValues[2] = {0,0};
                 revcpy<8>(reinterpret_cast<char*>(tempValues), readBuf, 2);
-
+                if (tempValues[0] < 0 || tempValues[1] < 0)
+                {
+#ifdef __EXCEPTIONS
+                    throw runtime_error("ERROR: negative value in the catalog");
+#else
+                    gLog << ___err ___ << "ERROR: negative value in the catalog" << endl;
+                    return;
+#endif
+                }
                 //add catalog entry
                 fCatalog[i].emplace_back(tempValues[0], tempValues[1]);
             }
 
+        //compute the total size of each compressed tile
+        fTileSize.resize(fNumTiles);
+        fTileOffsets.resize(fNumTiles);
+        for (uint32_t i=0;i<fNumTiles;i++)
+        {
+            fTileSize[i] = 0;
+            for (uint32_t j=0;j<fTable.num_cols;j++)
+            {
+                fTileSize[i] += fCatalog[i][j].first;
+                fTileOffsets[i].emplace_back(fCatalog[i][j].second - fCatalog[i][0].second);
+            }
+        }
         //see if there is a gap before heap data
         fHeapOff = tellg()+fTable.GetHeapShift();
-    }
-
+
+        if (!fCopy.is_open())
+            return;
+
+        //write catalog and heap gap to target file
+        seekg(catalogStart);
+
+        const size_t catSize = fTable.GetHeapShift() + fTable.total_bytes;
+
+        vector<char> buf(catSize);
+        read(buf.data(), catSize);
+
+        fCopy.write(buf.data(), catSize);
+        if (!fCopy)
+            clear(rdstate()|ios::badbit);
+    }
+    //overrides fits.h method with empty one
+    //work is done in ReadBinaryRow because it requires volatile data from ReadBinaryRow
+    virtual void WriteRowToCopyFile(size_t )
+    {
+
+    }
     // Compressed versin of the read row
     bool ReadBinaryRow(const size_t &rowNum, char *bufferToRead)
@@ -164 +212 @@
         const uint32_t requestedTile = rowNum/fNumRowsPerTile;
         const uint32_t currentTile   = fCurrentRow/fNumRowsPerTile;
+        const size_t   previousRow   = fCurrentRow;
 
         fCurrentRow = rowNum;
@@ -171 +220 @@
         {
             //read yet another chunk from the file
-            //the size that we should read is in the catalog. we should sum up the sizes of all columns
-            const uint32_t currentCatRow = fCurrentRow/fNumRowsPerTile;
-
-            int64_t sizeToRead  = 0;
-            for (uint32_t i=0;i<fCatalog[currentCatRow].size();i++)
-                sizeToRead += fCatalog[currentCatRow][i].first;
+            const int64_t sizeToRead = fTileSize[requestedTile];
 
             //skip to the beginning of the tile
-            seekg(fHeapOff+fCatalog[currentCatRow][0].second);
+            seekg(fHeapOff+fCatalog[requestedTile][0].second);
             read(fCompressedBuffer.data(), sizeToRead);
 
+            if (fCurrentRow == previousRow+1 &&
+                fCopy.is_open() &&
+                fCopy.good())
+            {
+                fCopy.write(fCompressedBuffer.data(), sizeToRead);
+                if (!fCopy)
+                    clear(rdstate()|ios::badbit);
+            }
+            else
+                if (fCopy.is_open())
+                    clear(rdstate()|ios::badbit);
+
             const uint32_t thisRoundNumRows = (GetNumRows()<fCurrentRow + fNumRowsPerTile) ? GetNumRows()%fNumRowsPerTile : fNumRowsPerTile;
 
             //uncompress it
-            UncompressBuffer(currentCatRow, thisRoundNumRows);
+            UncompressBuffer(requestedTile, thisRoundNumRows);
 
             // pointer to column (source buffer)
@@ -301 +357 @@
 
             //get the compression flag
-            const int64_t compressedOffset = fCatalog[catalogCurrentRow][i].second - fCatalog[catalogCurrentRow][0].second;
+            const int64_t compressedOffset = fTileOffsets[catalogCurrentRow][i];//fCatalog[catalogCurrentRow][i].second - fCatalog[catalogCurrentRow][0].second;
             const char    compressedFlag   = fCompressedBuffer[compressedOffset];