Index: trunk/FACT++/src/fitsCompressor.cc =================================================================== --- trunk/FACT++/src/fitsCompressor.cc (revision 16442) +++ trunk/FACT++/src/fitsCompressor.cc (revision 16443) @@ -633,5 +633,5 @@ { _transposedBuffer[i] = new char[_rowWidth*_numRowsPerTile]; - _compressedBuffer[i] = new char[_rowWidth*_numRowsPerTile + 1024*1024]; //use a bit more memory, in case the compression algorithms uses more + _compressedBuffer[i] = new char[_rowWidth*_numRowsPerTile + _columns.size()]; //use a bit more memory for compression flags if (_transposedBuffer[i] == NULL || _compressedBuffer[i] == NULL) return false; @@ -736,5 +736,5 @@ header.push_back(HeaderEntry("CHECKSUM", "'0000000000000000' ", "Checksum for the whole HDU")); header.push_back(HeaderEntry("DATASUM" , " 0" , "Checksum for the data block")); - header.push_back(HeaderEntry("EXTNAME" , "'DrsCalib' ", "name of this binary table extension")); + header.push_back(HeaderEntry("EXTNAME" , "'IntCalibration' ", "name of this binary table extension")); header.push_back(HeaderEntry("TTYPE1" , "'OffsetCalibration' ", "label for field 1")); header.push_back(HeaderEntry("TFORM1" , "'1474560I' ", "data format of field: 2-byte INTEGER")); @@ -1071,5 +1071,5 @@ setHeaderKey(HeaderEntry("TFIELDS", _columns.size(), "number of fields in each row")); int64_t heapSize = 0; - uint32_t compressedOffset = 0; + int64_t compressedOffset = 0; for (uint32_t i=0;i<_catalog.size();i++) for (uint32_t j=0;j<_catalog[i].size();j++) @@ -1077,4 +1077,5 @@ heapSize += _catalog[i][j].first; //set the catalog offsets to their actual values + if (compressedOffset < 0) return false; _catalog[i][j].second = compressedOffset; compressedOffset += _catalog[i][j].first; @@ -1198,6 +1199,11 @@ previousHuffmanSize = huffmanOutput.size(); } - memcpy(&dest[huffmanOffset], huffmanOutput.data(), huffmanOutput.size()); - return huffmanOutput.size() + huffmanOffset; + const size_t totalSize = huffmanOutput.size() + huffmanOffset; + + //only copy if not larger than not-compressed size + if (totalSize < numRows*sizeOfElems*numRowElems) + memcpy(&dest[huffmanOffset], huffmanOutput.data(), huffmanOutput.size()); + + return totalSize; } @@ -1731,5 +1737,6 @@ for (int j=0;j<1440;j++) { - int thisStartCell = reinterpret_cast(&buffer[startCellOffset])[j]; + const int thisStartCell = reinterpret_cast(&buffer[startCellOffset])[j]; + if (thisStartCell > -1) for (int k=0;k(&buffer[dataOffset])[numSlices*j + k] -= drsCalib16[1024*j + (thisStartCell+k)%1024]; @@ -1745,5 +1752,10 @@ inFile.close(); - outFile.close(); + if (!outFile.close()) + { + cout << "Something went wrong while writing the catalog: negative index" << endl; + return false; + } + delete[] drsCalib16; @@ -1762,5 +1774,8 @@ //get a compressed reader - factfits verifyFile(fileNameOut, tableName, false); +//TEMP try to copy the file too +// string copyName("/scratch/copyFile.fz"); + string copyName(""); + factfits verifyFile(fileNameOut, copyName, tableName, false); //and the header of the compressed file Index: trunk/Mars/mcore/factfits.h =================================================================== --- trunk/Mars/mcore/factfits.h (revision 16442) +++ trunk/Mars/mcore/factfits.h (revision 16443) @@ -21,5 +21,9 @@ // Default constructor factfits(const string& fname, const string& tableName="", bool force=false) : - zfits(fname, tableName, force) + zfits(fname, tableName, force), + fOffsetCalibration(0), + fOffsetStartCellData(0), + fOffsetData(0), + fNumRoi(0) { if (init()) @@ -29,5 +33,9 @@ // Alternative constructor factfits(const string& fname, const string& fout, const string& tableName, bool force=false) : - zfits(fname, fout, tableName, force) + zfits(fname, fout, tableName, force), + fOffsetCalibration(0), + fOffsetStartCellData(0), + fOffsetData(0), + fNumRoi(0) { if (init()) @@ -35,41 +43,26 @@ } - // -#if !defined(__MARS__) && !defined(__CINT__) - bool GetRow(size_t row, bool check=true) -#else - bool GetRowNum(size_t row, bool check=true) -#endif +private: + + void StageRow(size_t row, char* dest) { -#if !defined(__MARS__) && !defined(__CINT__) - if (!zfits::GetRow(row, check)) - return false; -#else - if (!zfits::GetRowNum(row, check)) - return false; -#endif + zfits::StageRow(row, dest); // This file does not contain fact data or no calibration to be applied - if (fOffsetCalibration.empty()) - return true; + if (fOffsetCalibration.empty()) + return; - //re-get the pointer to the data to access the offsets - const uint8_t offset = (row*fTable.bytes_per_row)%4; + //re-get the pointer to the data to access the offsets + const uint8_t offset = (row*fTable.bytes_per_row)%4; - int16_t *startCell = reinterpret_cast(fBufferRow.data() + offset + fOffsetStartCellData); + int16_t *startCell = reinterpret_cast(fBufferRow.data() + offset + fOffsetStartCellData); + int16_t *data = reinterpret_cast(fBufferRow.data() + offset + fOffsetData); - const Pointers::iterator dtaIt = fPointers.find("Data"); - if (dtaIt == fPointers.end()) return true; - int16_t *data = reinterpret_cast(dtaIt->second); - - for (uint32_t i=0; i<1440*1024; i+=1024, startCell++) - { - for (uint32_t j=0; j -1) + for (uint32_t j=0; j fOffsetCalibration; ///< integer values of the drs calibration used for compression + size_t fOffsetStartCellData; size_t fOffsetData; - size_t fOffsetStartCellData; uint16_t fNumRoi; Index: trunk/Mars/mcore/fits.h =================================================================== --- trunk/Mars/mcore/fits.h (revision 16442) +++ trunk/Mars/mcore/fits.h (revision 16443) @@ -612,6 +612,8 @@ peek(); if (eof() && !bad() && !tableName.empty()) + { + cout << "END OF FILE !" << endl; break; - + } // FIXME: Set limit on memory consumption const int rc = ReadBlock(block); @@ -741,4 +743,20 @@ } + virtual void WriteRowToCopyFile(size_t row) + { + if (row==fRow+1 && !fTable.isCompressed) + { + const uint8_t offset = (row*fTable.bytes_per_row)%4; + + fChkData.add(fBufferRow); + if (fCopy.is_open() && fCopy.good()) + fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row); + if (!fCopy) + clear(rdstate()|ios::badbit); + } + else + if (fCopy.is_open()) + clear(rdstate()|ios::badbit); + } uint8_t ReadRow(size_t row) { @@ -764,15 +782,5 @@ StageRow(row, fBufferRow.data()+offset); - if (row==fRow+1) - { - fChkData.add(fBufferRow); - if (fCopy.is_open() && fCopy.good()) - fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row); - if (!fCopy) - clear(rdstate()|ios::badbit); - } - else - if (fCopy.is_open()) - clear(rdstate()|ios::badbit); + WriteRowToCopyFile(row); fRow = row; Index: trunk/Mars/mcore/zfits.h =================================================================== --- trunk/Mars/mcore/zfits.h (revision 16442) +++ trunk/Mars/mcore/zfits.h (revision 16443) @@ -33,5 +33,6 @@ fNumRowsPerTile(0), fCurrentRow(-1), - fHeapOff(0) + fHeapOff(0), + fTileSize(0) { InitCompressionReading(); @@ -47,5 +48,6 @@ fNumRowsPerTile(0), fCurrentRow(-1), - fHeapOff(0) + fHeapOff(0), + fTileSize(0) { InitCompressionReading(); @@ -60,4 +62,18 @@ fRow++; return true; + } +protected: + + // Stage the requested row to internal buffer + // Does NOT return data to users + virtual void StageRow(size_t row, char* dest) + { + if (!fTable.isCompressed) + { + fits::StageRow(row, dest); + return; + } + + ReadBinaryRow(row, dest); } @@ -82,17 +98,4 @@ } - // Stage the requested row to internal buffer - // Does NOT return data to users - void StageRow(size_t row, char* dest) - { - if (!fTable.isCompressed) - { - fits::StageRow(row, dest); - return; - } - - ReadBinaryRow(row, dest); - } - // Copy decompressed data to location requested by user void MoveColumnDataToUserSpace(char* dest, const char* src, const Table::Column& c) @@ -118,4 +121,6 @@ vector > > fCatalog;///< Catalog, i.e. the main table that points to the compressed data. + vector fTileSize; ///< size in bytes of each compressed tile + vector > fTileOffsets; ///< offset from start of tile of a given compressed column void AllocateBuffers() @@ -127,5 +132,5 @@ fTransposedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile); - fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + 1024*1024); //use a bit more memory, in case the compression algorithms uses more + fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + fTable.num_cols); //use a bit more memory for compression flags } @@ -139,4 +144,7 @@ fCatalog.resize(fNumTiles); + const streampos catalogStart = tellg(); + + //do the actual reading for (uint32_t i=0;i(reinterpret_cast(tempValues), readBuf, 2); - + if (tempValues[0] < 0 || tempValues[1] < 0) + { +#ifdef __EXCEPTIONS + throw runtime_error("ERROR: negative value in the catalog"); +#else + gLog << ___err ___ << "ERROR: negative value in the catalog" << endl; + return; +#endif + } //add catalog entry fCatalog[i].emplace_back(tempValues[0], tempValues[1]); } + //compute the total size of each compressed tile + fTileSize.resize(fNumTiles); + fTileOffsets.resize(fNumTiles); + for (uint32_t i=0;i buf(catSize); + read(buf.data(), catSize); + + fCopy.write(buf.data(), catSize); + if (!fCopy) + clear(rdstate()|ios::badbit); + } + //overrides fits.h method with empty one + //work is done in ReadBinaryRow because it requires volatile data from ReadBinaryRow + virtual void WriteRowToCopyFile(size_t ) + { + + } // Compressed versin of the read row bool ReadBinaryRow(const size_t &rowNum, char *bufferToRead) @@ -164,4 +212,5 @@ const uint32_t requestedTile = rowNum/fNumRowsPerTile; const uint32_t currentTile = fCurrentRow/fNumRowsPerTile; + const size_t previousRow = fCurrentRow; fCurrentRow = rowNum; @@ -171,19 +220,26 @@ { //read yet another chunk from the file - //the size that we should read is in the catalog. we should sum up the sizes of all columns - const uint32_t currentCatRow = fCurrentRow/fNumRowsPerTile; - - int64_t sizeToRead = 0; - for (uint32_t i=0;i