Changeset 16443 for trunk

Timestamp:

05/29/13 22:02:43 (11 years ago)

Author:

lyard

Message:

more tweaks to factfits

Location:

trunk

Files:

• FACT++/src/fitsCompressor.cc (modified) (8 diffs)
• Mars/mcore/factfits.h (modified) (6 diffs)
• Mars/mcore/fits.h (modified) (3 diffs)
• Mars/mcore/zfits.h (modified) (11 diffs)

trunk/FACT++/src/fitsCompressor.cc

@@ -633 +633 @@
     {
         _transposedBuffer[i] = new char[_rowWidth*_numRowsPerTile];
-        _compressedBuffer[i] = new char[_rowWidth*_numRowsPerTile + 1024*1024]; //use a bit more memory, in case the compression algorithms uses more
+        _compressedBuffer[i] = new char[_rowWidth*_numRowsPerTile + _columns.size()]; //use a bit more memory for compression flags
         if (_transposedBuffer[i] == NULL || _compressedBuffer[i] == NULL)
             return false;
@@ -736 +736 @@
     header.push_back(HeaderEntry("CHECKSUM", "'0000000000000000'  ", "Checksum for the whole HDU"));
     header.push_back(HeaderEntry("DATASUM" ,  "         0"         , "Checksum for the data block"));
-    header.push_back(HeaderEntry("EXTNAME" , "'DrsCalib'          ", "name of this binary table extension"));
+    header.push_back(HeaderEntry("EXTNAME" , "'IntCalibration'    ", "name of this binary table extension"));
     header.push_back(HeaderEntry("TTYPE1"  , "'OffsetCalibration' ", "label for field   1"));
     header.push_back(HeaderEntry("TFORM1"  , "'1474560I'          ", "data format of field: 2-byte INTEGER"));
@@ -1071 +1071 @@
         setHeaderKey(HeaderEntry("TFIELDS", _columns.size(), "number of fields in each row"));
         int64_t heapSize = 0;
-        uint32_t compressedOffset = 0;
+        int64_t compressedOffset = 0;
         for (uint32_t i=0;i<_catalog.size();i++)
             for (uint32_t j=0;j<_catalog[i].size();j++)
@@ -1077 +1077 @@
                 heapSize += _catalog[i][j].first;
                 //set the catalog offsets to their actual values
+                if (compressedOffset < 0) return false;
                 _catalog[i][j].second = compressedOffset;
                 compressedOffset += _catalog[i][j].first;
@@ -1198 +1199 @@
         previousHuffmanSize = huffmanOutput.size();
     }
-    memcpy(&dest[huffmanOffset], huffmanOutput.data(), huffmanOutput.size());
-    return huffmanOutput.size() + huffmanOffset;
+    const size_t totalSize = huffmanOutput.size() + huffmanOffset;
+
+    //only copy if not larger than not-compressed size
+    if (totalSize < numRows*sizeOfElems*numRowElems)
+        memcpy(&dest[huffmanOffset], huffmanOutput.data(), huffmanOutput.size());
+
+    return totalSize;
 }
 
@@ -1731 +1737 @@
             for (int j=0;j<1440;j++)
             {
-                int thisStartCell = reinterpret_cast<int16_t*>(&buffer[startCellOffset])[j];
+                const int thisStartCell = reinterpret_cast<int16_t*>(&buffer[startCellOffset])[j];
+                if (thisStartCell > -1)
                 for (int k=0;k<numSlices;k++)
                     reinterpret_cast<int16_t*>(&buffer[dataOffset])[numSlices*j + k] -= drsCalib16[1024*j + (thisStartCell+k)%1024];
@@ -1745 +1752 @@
 
     inFile.close();
-    outFile.close();
+    if (!outFile.close())
+    {
+        cout << "Something went wrong while writing the catalog: negative index" << endl;
+        return false;
+    }
+
     delete[] drsCalib16;
 
@@ -1762 +1774 @@
 
     //get a compressed reader
-    factfits verifyFile(fileNameOut, tableName, false);
+//TEMP try to copy the file too
+//    string copyName("/scratch/copyFile.fz");
+    string copyName("");
+    factfits verifyFile(fileNameOut, copyName, tableName, false);
 
     //and the header of the compressed file

trunk/Mars/mcore/factfits.h

@@ -21 +21 @@
     // Default constructor
     factfits(const string& fname, const string& tableName="", bool force=false) :
-        zfits(fname, tableName, force)
+        zfits(fname, tableName, force),
+        fOffsetCalibration(0),
+        fOffsetStartCellData(0),
+        fOffsetData(0),
+        fNumRoi(0)
     {
         if (init())
@@ -29 +33 @@
     // Alternative constructor
     factfits(const string& fname, const string& fout, const string& tableName, bool force=false) :
-        zfits(fname, fout, tableName, force)
+        zfits(fname, fout, tableName, force),
+        fOffsetCalibration(0),
+        fOffsetStartCellData(0),
+        fOffsetData(0),
+        fNumRoi(0)
     {
         if (init())
@@ -35 +43 @@
     }
 
-    //
-#if !defined(__MARS__) && !defined(__CINT__)
-    bool GetRow(size_t row, bool check=true)
-#else
-    bool GetRowNum(size_t row, bool check=true)
-#endif
+private:
+
+    void StageRow(size_t row, char* dest)
     {
-#if !defined(__MARS__) && !defined(__CINT__)
-        if (!zfits::GetRow(row, check))
-            return false;
-#else
-        if (!zfits::GetRowNum(row, check))
-            return false;
-#endif
+        zfits::StageRow(row, dest);
         // This file does not contain fact data or no calibration to be applied
-        if (fOffsetCalibration.empty())
-            return true;
+         if (fOffsetCalibration.empty())
+             return;
 
-        //re-get the pointer to the data to access the offsets
-        const uint8_t offset = (row*fTable.bytes_per_row)%4;
+         //re-get the pointer to the data to access the offsets
+         const uint8_t offset = (row*fTable.bytes_per_row)%4;
 
-        int16_t *startCell = reinterpret_cast<int16_t*>(fBufferRow.data() + offset + fOffsetStartCellData);
+         int16_t *startCell = reinterpret_cast<int16_t*>(fBufferRow.data() + offset + fOffsetStartCellData);
+         int16_t *data      = reinterpret_cast<int16_t*>(fBufferRow.data() + offset + fOffsetData);
 
-        const Pointers::iterator dtaIt = fPointers.find("Data");
-        if (dtaIt == fPointers.end()) return true;
-        int16_t *data      = reinterpret_cast<int16_t*>(dtaIt->second);
-
-        for (uint32_t i=0; i<1440*1024; i+=1024, startCell++)
-        {
-            for (uint32_t j=0; j<fNumRoi; j++, data++)
-                *data += fOffsetCalibration[i + (*startCell+j)%1024];
-        }
-
-        return true;
+         for (uint32_t i=0; i<1440*1024; i+=1024, startCell++)
+         {
+             if ((*startCell) > -1)
+             for (uint32_t j=0; j<fNumRoi; j++, data++)
+                 *data += fOffsetCalibration[i + (*startCell+j)%1024];
+         }
     }
-
-private:
 
     bool init()
@@ -104 +97 @@
     void readDrsCalib(const string& fileName)
     {
-        zfits calib(fileName, "DrsCalib");
+        //should not be mandatory, but improves the perfs a lot when reading not compressed, gzipped files
+        if (!IsCompressedFITS())
+            return;
+
+        zfits calib(fileName, "IntCalibration");
+
         if (calib.bad())
         {
@@ -119 +117 @@
             clear(rdstate()|ios::badbit);
 #ifdef __EXCEPTIONS
-            throw runtime_error("Table 'DrsCalib' found, but not with one row as expected");
+            throw runtime_error("Table 'IntCalibration' found, but not with one row as expected");
 #else
-            gLog << ___err___ << "ERROR - Table 'DrsCalib' found, but not with one row as expected" << endl;
+            gLog << ___err___ << "ERROR - Table 'IntCalibration' found, but not with one row as expected" << endl;
             return;
 #endif
@@ -164 +162 @@
     vector<int16_t> fOffsetCalibration; ///< integer values of the drs calibration used for compression
 
+    size_t fOffsetStartCellData;
     size_t fOffsetData;
-    size_t fOffsetStartCellData;
 
     uint16_t fNumRoi;

trunk/Mars/mcore/fits.h

@@ -612 +612 @@
                 peek();
                 if (eof() && !bad() && !tableName.empty())
+                {
+                    cout << "END OF FILE !" << endl;
                     break;
-
+                }
                 // FIXME: Set limit on memory consumption
                 const int rc = ReadBlock(block);
@@ -741 +743 @@
     }
 
+    virtual void WriteRowToCopyFile(size_t row)
+    {
+        if (row==fRow+1 && !fTable.isCompressed)
+        {
+            const uint8_t offset = (row*fTable.bytes_per_row)%4;
+
+            fChkData.add(fBufferRow);
+            if (fCopy.is_open() && fCopy.good())
+                fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row);
+            if (!fCopy)
+                clear(rdstate()|ios::badbit);
+        }
+        else
+            if (fCopy.is_open())
+                clear(rdstate()|ios::badbit);
+    }
     uint8_t ReadRow(size_t row)
     {
@@ -764 +782 @@
         StageRow(row, fBufferRow.data()+offset);
 
-        if (row==fRow+1)
-        {
-            fChkData.add(fBufferRow);
-            if (fCopy.is_open() && fCopy.good())
-                fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row);
-            if (!fCopy)
-                clear(rdstate()|ios::badbit);
-        }
-        else
-            if (fCopy.is_open())
-                clear(rdstate()|ios::badbit);
+        WriteRowToCopyFile(row);
 
         fRow = row;

trunk/Mars/mcore/zfits.h

@@ -33 +33 @@
         fNumRowsPerTile(0),
         fCurrentRow(-1),
-        fHeapOff(0)
+        fHeapOff(0),
+        fTileSize(0)
     {
         InitCompressionReading();
@@ -47 +48 @@
               fNumRowsPerTile(0),
               fCurrentRow(-1),
-              fHeapOff(0)
+              fHeapOff(0),
+              fTileSize(0)
     {
         InitCompressionReading();
@@ -60 +62 @@
         fRow++;
         return true;
+    }
+protected:
+
+    //  Stage the requested row to internal buffer
+    //  Does NOT return data to users
+    virtual void StageRow(size_t row, char* dest)
+    {
+        if (!fTable.isCompressed)
+        {
+            fits::StageRow(row, dest);
+            return;
+        }
+
+        ReadBinaryRow(row, dest);
     }
 
@@ -82 +98 @@
     }
 
-    //  Stage the requested row to internal buffer
-    //  Does NOT return data to users
-    void StageRow(size_t row, char* dest)
-    {
-        if (!fTable.isCompressed)
-        {
-            fits::StageRow(row, dest);
-            return;
-        }
-
-        ReadBinaryRow(row, dest);
-    }
-
     // Copy decompressed data to location requested by user
     void MoveColumnDataToUserSpace(char* dest, const char* src, const Table::Column& c)
@@ -118 +121 @@
 
     vector<vector<pair<int64_t, int64_t> > > fCatalog;///< Catalog, i.e. the main table that points to the compressed data.
+    vector<size_t> fTileSize; ///< size in bytes of each compressed tile
+    vector<vector<size_t> > fTileOffsets; ///< offset from start of tile of a given compressed column
 
     void AllocateBuffers()
@@ -127 +132 @@
 
         fTransposedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile);
-        fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + 1024*1024); //use a bit more memory, in case the compression algorithms uses more
+        fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + fTable.num_cols); //use a bit more memory for compression flags
     }
 
@@ -139 +144 @@
         fCatalog.resize(fNumTiles);
 
+        const streampos catalogStart = tellg();
+
+        //do the actual reading
         for (uint32_t i=0;i<fNumTiles;i++)
             for (uint32_t j=0;j<fTable.num_cols;j++)
@@ -147 +155 @@
                 int64_t tempValues[2] = {0,0};
                 revcpy<8>(reinterpret_cast<char*>(tempValues), readBuf, 2);
-
+                if (tempValues[0] < 0 || tempValues[1] < 0)
+                {
+#ifdef __EXCEPTIONS
+                    throw runtime_error("ERROR: negative value in the catalog");
+#else
+                    gLog << ___err ___ << "ERROR: negative value in the catalog" << endl;
+                    return;
+#endif
+                }
                 //add catalog entry
                 fCatalog[i].emplace_back(tempValues[0], tempValues[1]);
             }
 
+        //compute the total size of each compressed tile
+        fTileSize.resize(fNumTiles);
+        fTileOffsets.resize(fNumTiles);
+        for (uint32_t i=0;i<fNumTiles;i++)
+        {
+            fTileSize[i] = 0;
+            for (uint32_t j=0;j<fTable.num_cols;j++)
+            {
+                fTileSize[i] += fCatalog[i][j].first;
+                fTileOffsets[i].emplace_back(fCatalog[i][j].second - fCatalog[i][0].second);
+            }
+        }
         //see if there is a gap before heap data
         fHeapOff = tellg()+fTable.GetHeapShift();
-    }
-
+
+        if (!fCopy.is_open())
+            return;
+
+        //write catalog and heap gap to target file
+        seekg(catalogStart);
+
+        const size_t catSize = fTable.GetHeapShift() + fTable.total_bytes;
+
+        vector<char> buf(catSize);
+        read(buf.data(), catSize);
+
+        fCopy.write(buf.data(), catSize);
+        if (!fCopy)
+            clear(rdstate()|ios::badbit);
+    }
+    //overrides fits.h method with empty one
+    //work is done in ReadBinaryRow because it requires volatile data from ReadBinaryRow
+    virtual void WriteRowToCopyFile(size_t )
+    {
+
+    }
     // Compressed versin of the read row
     bool ReadBinaryRow(const size_t &rowNum, char *bufferToRead)
@@ -164 +212 @@
         const uint32_t requestedTile = rowNum/fNumRowsPerTile;
         const uint32_t currentTile   = fCurrentRow/fNumRowsPerTile;
+        const size_t   previousRow   = fCurrentRow;
 
         fCurrentRow = rowNum;
@@ -171 +220 @@
         {
             //read yet another chunk from the file
-            //the size that we should read is in the catalog. we should sum up the sizes of all columns
-            const uint32_t currentCatRow = fCurrentRow/fNumRowsPerTile;
-
-            int64_t sizeToRead  = 0;
-            for (uint32_t i=0;i<fCatalog[currentCatRow].size();i++)
-                sizeToRead += fCatalog[currentCatRow][i].first;
+            const int64_t sizeToRead = fTileSize[requestedTile];
 
             //skip to the beginning of the tile
-            seekg(fHeapOff+fCatalog[currentCatRow][0].second);
+            seekg(fHeapOff+fCatalog[requestedTile][0].second);
             read(fCompressedBuffer.data(), sizeToRead);
 
+            if (fCurrentRow == previousRow+1 &&
+                fCopy.is_open() &&
+                fCopy.good())
+            {
+                fCopy.write(fCompressedBuffer.data(), sizeToRead);
+                if (!fCopy)
+                    clear(rdstate()|ios::badbit);
+            }
+            else
+                if (fCopy.is_open())
+                    clear(rdstate()|ios::badbit);
+
             const uint32_t thisRoundNumRows = (GetNumRows()<fCurrentRow + fNumRowsPerTile) ? GetNumRows()%fNumRowsPerTile : fNumRowsPerTile;
 
             //uncompress it
-            UncompressBuffer(currentCatRow, thisRoundNumRows);
+            UncompressBuffer(requestedTile, thisRoundNumRows);
 
             // pointer to column (source buffer)
@@ -301 +357 @@
 
             //get the compression flag
-            const int64_t compressedOffset = fCatalog[catalogCurrentRow][i].second - fCatalog[catalogCurrentRow][0].second;
+            const int64_t compressedOffset = fTileOffsets[catalogCurrentRow][i];//fCatalog[catalogCurrentRow][i].second - fCatalog[catalogCurrentRow][0].second;
             const char    compressedFlag   = fCompressedBuffer[compressedOffset];