Changeset 16897

Timestamp:

06/24/13 15:57:53 (11 years ago)

Author:

lyard

Message:

Removed bytes swapping for RAWSUM

Location:

trunk

Files:

• FACT++/src/fitsCompressor.cc (modified) (18 diffs)
• Mars/mcore/checksum.h (modified) (4 diffs)
• Mars/mcore/zfits.h (modified) (4 diffs)

trunk/FACT++/src/fitsCompressor.cc

@@ -433 +433 @@
         bool writeBinaryRow(const char* bufferToWrite);
 
+        uint32_t getRowWidth();
+
         ///@brief assign a given (already loaded) drs calibration
         void setDrsCalib(int16_t* data);
@@ -627 +629 @@
     if (_buffer != NULL)
     {
+        _buffer = _buffer-4;
         delete[] _buffer;
         _buffer = NULL;
@@ -649 +652 @@
     if (_buffer)
     {
+        _buffer = _buffer - 4;
         delete[] _buffer;
         for (uint32_t i=0;i<_compressedBuffer.size();i++)
@@ -657 +661 @@
         }
     }
-    _buffer = new char[_rowWidth*_numRowsPerTile];
+    _buffer = new char[_rowWidth*_numRowsPerTile + 12];
     if (_buffer == NULL) return false;
+    memset(_buffer, 0, 4);
+    _buffer = _buffer + 4;
     if (_compressedBuffer.size() != _numThreads)
     {
@@ -667 +673 @@
     {
         _transposedBuffer[i] = new char[_rowWidth*_numRowsPerTile];
-        _compressedBuffer[i] = new char[_rowWidth*_numRowsPerTile + _columns.size() + sizeof(TileHeader)]; //use a bit more memory for compression flags
+        _compressedBuffer[i] = new char[_rowWidth*_numRowsPerTile + _columns.size() + sizeof(TileHeader) + 12]; //use a bit more memory for compression flags and checksumming
         if (_transposedBuffer[i] == NULL || _compressedBuffer[i] == NULL)
             return false;
@@ -1133 +1139 @@
         setHeaderKey(HeaderEntry("PCOUNT", 0, "size of special data area"));
     }
+    ostringstream str;
+
     writeHeader(true);
 
-    ostringstream str;
+    str.str("");
     str << _checksum.val();
 
@@ -1160 +1168 @@
 {
     uint32_t thisRoundNumRows = (_totalNumRows%_numRowsPerTile) ? _totalNumRows%_numRowsPerTile : _numRowsPerTile;
+
     //copy the tile and transpose it
     uint32_t offset = 0;
@@ -1215 +1224 @@
     }
     return _file.good();
+}
+
+uint32_t CompressedFitsWriter::getRowWidth()
+{
+    return _rowWidth;
 }
 
@@ -1642 +1656 @@
     //allocate the buffer for temporary storage of each read/written row
     uint32_t rowWidth = inFile.GetUInt("NAXIS1");
-    char* buffer = new char[rowWidth];
+    char* buffer = new char[rowWidth + 12];
+    memset(buffer, 0, 4);
+    buffer = buffer+4;
 
     //get the source columns
@@ -1651 +1667 @@
 
     //Add columns.
+    uint32_t totalRowWidth = 0;
     for (uint32_t i=0;i<sortedColumns.size(); i++)
     {
@@ -1674 +1691 @@
         smoothmanProcessings[1] = FACT_HUFFMAN16;
 //        smoothmanProcessings[2] = FACT_RAW;
+
+        totalRowWidth += sortedColumns[i].bytes;
 
         //first lets see if we do have an explicit request
@@ -1735 +1754 @@
 //        cout << i->first << endl;
     }
+
+    outFile.setHeaderKey(CompressedFitsFile::HeaderEntry("RAWSUM", "         0", "Checksum of raw littlen endian data"));
 
     //deal with the DRS calib
@@ -1826 +1847 @@
     }
 
+    Checksum rawsum;
     //Convert each row one after the other
     for (uint32_t i=0;i<inFile.GetNumRows();i++)
@@ -1838 +1860 @@
             count++;
         }
+
+        char* checkSumPointer = buffer;
+        const int chkOffset = (i*totalRowWidth)%4;
+        checkSumPointer -= chkOffset;
+        uint32_t sizeToChecksum = totalRowWidth + chkOffset;
+        if (sizeToChecksum%4 != 0)
+        {
+            int32_t extraBytes = 4 - (sizeToChecksum%4);
+            memset(checkSumPointer+sizeToChecksum, 0, extraBytes);
+            sizeToChecksum += extraBytes;
+        }
+        rawsum.add(checkSumPointer, sizeToChecksum, false);
 
         if (startCellOffset != -1)
@@ -1851 +1885 @@
         outFile.writeBinaryRow(buffer);
     };
+    ostringstream strSum;
+    strSum << rawsum.val();
+    outFile.setHeaderKey(CompressedFitsFile::HeaderEntry("RAWSUM", strSum.str(), "Checksum of raw littlen endian data"));
 
     //Get table name for later use in case the compressed file is to be verified
@@ -1927 +1964 @@
             continue;
         }
+        if (k == "RAWSUM")
+        {
+            continue;
+        }
 
         if (k == "ZDTASUM")
@@ -2028 +2069 @@
 //    verifyFile.close();
     if (!verifyFile.IsFileOk())
-        cout << "ERROR: file checksum seems wrong" << endl;
+        cout << "ERROR: file checksums seems wrong" << endl;
 
     reconstructedFile.close();
@@ -2051 +2092 @@
     }
 
+    buffer = buffer-4;
     delete[] buffer;
     return 0;
-
-}
-
+}
+

trunk/Mars/mcore/checksum.h

@@ -47 +47 @@
 
 
-    bool add(const char *buf, size_t len)
+    bool add(const char *buf, size_t len, bool big_endian = true)
     {
         // Avoid overflows in carry bits
@@ -73 +73 @@
         uint32_t *hilo  = reinterpret_cast<uint32_t*>(&buffer);
 
+
+        const uint16_t *end = sbuf + len/2;
+
+        if (big_endian)
+            addLoopSwapping(sbuf, end, hilo);
+        else
+            addLoop(sbuf, end, hilo);
+        /*const uint16_t *end = sbuf + len/2;
+        while (1)
+        {
+            if (sbuf==end)
+                break;
+
+            hilo[0] += ntohs(*sbuf++);
+
+            if (sbuf==end)
+                break;
+
+            hilo[1] += ntohs(*sbuf++);
+        }*/
+
+        HandleCarryBits();
+
+        return true;
+    }
+
+    void addLoopSwapping(const uint16_t *sbuf, const uint16_t *end, uint32_t* hilo)
+    {
         /*
         for (size_t i = 0; i < len/2; i++)
@@ -83 +111 @@
         // This is about as twice as fast as the loop above
         // ntohs is CPU optimized, i%2 doesn't need to be computed
-        const uint16_t *end = sbuf + len/2;
         while (1)
         {
@@ -96 +123 @@
             hilo[1] += ntohs(*sbuf++);
         }
-
-        HandleCarryBits();
-
-        return true;
-    }
-
-    bool add(const vector<char> &v)
-    {
-        return add(v.data(), v.size());
+    }
+
+    void addLoop(const uint16_t *sbuf, const uint16_t *end, uint32_t* hilo)
+    {
+        while (1)
+        {
+            if (sbuf==end)
+                break;
+
+            hilo[0] += ntohs(*sbuf++);
+
+            if (sbuf==end)
+                break;
+
+            hilo[1] += ntohs(*sbuf++);
+        }
+    }
+
+    bool add(const vector<char> &v, bool big_endian = true)
+    {
+        return add(v.data(), v.size(), big_endian);
     }
 

trunk/Mars/mcore/zfits.h

@@ -70 +70 @@
     }
 
+    virtual bool IsFileOk() const
+    {
+        bool rawsum = true;
+
+        if (HasKey("RAWSUM"))
+        {
+                ostringstream str;
+                str << fRawsum.val();
+                rawsum = (GetStr("RAWSUM") == str.str());
+        }
+
+        return fits::IsFileOk() && rawsum;
+    };
+
 protected:
 
@@ -130 +144 @@
 
         if (fTable.is_compressed)
-        {
-            for (auto it=fTable.sorted_cols.begin(); it!= fTable.sorted_cols.end(); it++)
+        for (auto it=fTable.sorted_cols.begin(); it!= fTable.sorted_cols.end(); it++)
+            if (it->comp != kCompFACT)
             {
                 if (it->comp == kCompFACT)
@@ -190 +204 @@
     vector<size_t>                         fTileSize; ///< size in bytes of each compressed tile
     vector<vector<size_t>>                 fTileOffsets; ///< offset from start of tile of a given compressed column
+
+    Checksum fRawsum;   ///< Checksum of the uncompressed, raw data
 
     // Get buffer space
@@ -270 +286 @@
     //overrides fits.h method with empty one
     //work is done in ReadBinaryRow because it requires volatile data from ReadBinaryRow
-    virtual void WriteRowToCopyFile(size_t )
-    {
-
+    virtual void WriteRowToCopyFile(size_t row)
+    {
+        if (row == fRow+1)
+            fRawsum.add(fBufferRow, false);
     }