Changeset 16882 for trunk/Mars/mcore

Timestamp:

06/21/13 17:08:18 (11 years ago)

Author:

lyard

Message:

added correct checksum calculation for compressed fits reader (zfits)

Location:

trunk/Mars/mcore

Files:

• fits.h (modified) (2 diffs)
• zfits.h (modified) (11 diffs)

trunk/Mars/mcore/fits.h

@@ -758 +758 @@
                 clear(rdstate()|ios::badbit);
     }
-    uint8_t ReadRow(size_t row)
-    {
-        // For the checksum we need everything to be correctly aligned
-        const uint8_t offset = (row*fTable.bytes_per_row)%4;
-
-        auto ib = fBufferRow.begin();
-        auto ie = fBufferRow.end();
+
+    void ZeroBufferForChecksum(vector<char>& vec, const uint64_t extraZeros=0)
+    {
+        auto ib = vec.begin();
+        auto ie = vec.end();
+
         *ib++ = 0;
         *ib++ = 0;
@@ -770 +769 @@
         *ib   = 0;
 
-        *--ie = 0;
-        *--ie = 0;
-        *--ie = 0;
-        *--ie = 0;
-        *--ie = 0;
-        *--ie = 0;
-        *--ie = 0;
-        *--ie = 0;
+        for (uint64_t i=0;i<extraZeros+8;i++)
+            *--ie = 0;
+    }
+
+    uint8_t ReadRow(size_t row)
+    {
+        // For the checksum we need everything to be correctly aligned
+        const uint8_t offset = (row*fTable.bytes_per_row)%4;
+
+        ZeroBufferForChecksum(fBufferRow);
 
         StageRow(row, fBufferRow.data()+offset);

trunk/Mars/mcore/zfits.h

@@ -188 +188 @@
     size_t fNumTiles;       ///< Total number of tiles
     size_t fNumRowsPerTile; ///< Number of rows per compressed tile
-    size_t fCurrentRow;     ///< current row in memory.
+    int64_t fCurrentRow;    ///< current row in memory signed because we need -1
 
     streamoff fHeapOff;     ///< offset from the beginning of the file of the binary data
+    streamoff fHeapFromDataStart; ///< offset from the beginning of the data table
 
     vector<vector<pair<int64_t, int64_t> > > fCatalog;///< Catalog, i.e. the main table that points to the compressed data.
@@ -202 +203 @@
 
         fTransposedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile);
-        fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + fTable.num_cols*(sizeof(BlockHeader)+256)); //use a bit more memory for block headers
+        fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile +
+                                 fTable.num_cols*(sizeof(BlockHeader)+256) + //use a bit more memory for block headers. 256 char coding the compression sequence max.
+                                 sizeof(TileHeader), //a bit more for the tile headers
+                                 8); //and a bit more for checksuming
     }
 
@@ -208 +212 @@
     void ReadCatalog()
     {
-        char readBuf[16];
+        vector<char> readBuf(16);
         fCatalog.resize(fNumTiles);
 
         const streampos catalogStart = tellg();
+
+fChkData.reset();
 
         //do the actual reading
@@ -217 +223 @@
             for (uint32_t j=0;j<fTable.num_cols;j++)
             {
-                read(readBuf, 2*sizeof(int64_t));
-
+                read(readBuf.data(), 2*sizeof(int64_t));
+                fChkData.add(readBuf);
                 //swap the bytes
                 int64_t tempValues[2] = {0,0};
-                revcpy<8>(reinterpret_cast<char*>(tempValues), readBuf, 2);
+                revcpy<8>(reinterpret_cast<char*>(tempValues), readBuf.data(), 2);
                 if (tempValues[0] < 0 || tempValues[1] < 0)
                 {
@@ -250 +256 @@
         //see if there is a gap before heap data
         fHeapOff = tellg()+fTable.GetHeapShift();
+        fHeapFromDataStart = fNumTiles*fTable.num_cols*2*sizeof(int64_t) + fTable.GetHeapShift();
 
         if (!fCopy.is_open())
@@ -269 +276 @@
     //overrides fits.h method with empty one
     //work is done in ReadBinaryRow because it requires volatile data from ReadBinaryRow
-    virtual void WriteRowToCopyFile(size_t row)
-    {
-        if (row == fRow+1)
-            fChkData.add(fBufferRow);
+    virtual void WriteRowToCopyFile(size_t )
+    {
+
     }
 
@@ -284 +290 @@
         const uint32_t currentTile   = fCurrentRow/fNumRowsPerTile;
 
+        bool addCheckSum = ((requestedTile == currentTile+1) || (fCurrentRow == -1));
+
         fCurrentRow = rowNum;
-
         //should we read yet another chunk of data ?
         if (requestedTile != currentTile)
         {
             //read yet another chunk from the file
-            const int64_t sizeToRead = fTileSize[requestedTile];
+            const int64_t sizeToRead = fTileSize[requestedTile] + sizeof(TileHeader);
 
             //skip to the beginning of the tile
-            seekg(fHeapOff+fCatalog[requestedTile][0].second - sizeof(TileHeader));
-            TileHeader tHead;
-            read((char*)(&tHead), sizeof(TileHeader));
-            read(fCompressedBuffer.data(), sizeToRead);
+            const int64_t tileStart =  fCatalog[requestedTile][0].second - sizeof(TileHeader);
+
+            seekg(fHeapOff+tileStart);
+
+            //calculate the 32 bits offset of the current tile.
+            const uint32_t offset = (tileStart + fHeapFromDataStart)%4;
+
+            //point the tile header where it should be
+            //we ain't checking the header now
+//            TileHeader* tHead = reinterpret_cast<TileHeader*>(fCompressedBuffer.data()+offset);
+
+            ZeroBufferForChecksum(fCompressedBuffer, fCompressedBuffer.size()-(sizeToRead+offset+8));
+
+            //read one tile from disk
+            read(fCompressedBuffer.data()+offset, sizeToRead);
+
+            if (addCheckSum)
+                fChkData.add(fCompressedBuffer);
 
             if (requestedTile == currentTile+1 &&
@@ -302 +323 @@
                 fCopy.good())
             {
-                fCopy.write((char*)(&tHead), sizeof(TileHeader));
-                fCopy.write(fCompressedBuffer.data(), sizeToRead);
+                fCopy.write(fCompressedBuffer.data()+offset, sizeToRead);
                 if (!fCopy)
                     clear(rdstate()|ios::badbit);
@@ -314 +334 @@
 
             //uncompress it
-            UncompressBuffer(requestedTile, thisRoundNumRows);
+            UncompressBuffer(requestedTile, thisRoundNumRows, offset+sizeof(TileHeader));
 
             // pointer to column (source buffer)
@@ -413 +433 @@
     // Data has been read from disk. Uncompress it !
     void UncompressBuffer(const uint32_t &catalogCurrentRow,
-                          const uint32_t &thisRoundNumRows)
+                          const uint32_t &thisRoundNumRows,
+                          const uint32_t offset)
     {
         char *dest = fTransposedBuffer.data();
@@ -425 +446 @@
 
             //get the compression flag
-            const int64_t compressedOffset = fTileOffsets[catalogCurrentRow][i];
+            const int64_t compressedOffset = fTileOffsets[catalogCurrentRow][i]+offset;
 
             const BlockHeader* head = reinterpret_cast<BlockHeader*>(&fCompressedBuffer[compressedOffset]);