Index: /trunk/Mars/mcore/zfits.h =================================================================== --- /trunk/Mars/mcore/zfits.h (revision 17260) +++ /trunk/Mars/mcore/zfits.h (revision 17261) @@ -14,6 +14,4 @@ #include "FITS.h" -using namespace FITS; - #ifndef __MARS__ namespace std @@ -26,10 +24,6 @@ // Basic constructor - zfits(const string& fname, const string& tableName="", bool force=false) : fits(), - fNumTiles(0), - fNumRowsPerTile(0), - fCurrentRow(-1), - fHeapOff(0), - fTileSize(0) + zfits(const string& fname, const string& tableName="", bool force=false) + : fNumTiles(0), fNumRowsPerTile(0), fCurrentRow(-1), fHeapOff(0), fTileSize(0) { open(fname.c_str()); @@ -39,10 +33,6 @@ // Alternative contstructor - zfits(const string& fname, const string& fout, const string& tableName, bool force=false) : fits(), - fNumTiles(0), - fNumRowsPerTile(0), - fCurrentRow(-1), - fHeapOff(0), - fTileSize(0) + zfits(const string& fname, const string& fout, const string& tableName, bool force=false) + : fNumTiles(0), fNumRowsPerTile(0), fCurrentRow(-1), fHeapOff(0), fTileSize(0) { open(fname.c_str()); @@ -67,5 +57,5 @@ if (HasKey("RAWSUM")) { - ostringstream str; + std::ostringstream str; str << fRawsum.val(); rawsum = (GetStr("RAWSUM") == str.str()); @@ -117,5 +107,5 @@ if (fTable.is_compressed) - for (auto it=fTable.sorted_cols.begin(); it!= fTable.sorted_cols.end(); it++) + for (auto it=fTable.sorted_cols.cbegin(); it!= fTable.sorted_cols.cend(); it++) { if (it->comp == kCompFACT) @@ -131,7 +121,6 @@ } - fColumnOrdering.resize(fTable.sorted_cols.size()); - for (auto it=fColumnOrdering.begin(); it != fColumnOrdering.end(); it++) - (*it) = kOrderByRow; + fColumnOrdering.resize(fTable.sorted_cols.size(), FITS::kOrderByRow); + //Get compressed specific keywords fNumTiles = fTable.is_compressed ? GetInt("NAXIS2") : 0; @@ -160,8 +149,8 @@ } - vector fBuffer; /// fTransposedBuffer; /// fCompressedBuffer; /// fColumnOrdering; ///< ordering of the column's rows. Can change from tile to tile. + std::vector fBuffer; /// fTransposedBuffer; /// fCompressedBuffer; /// fColumnOrdering; ///< ordering of the column's rows. Can change from tile to tile. size_t fNumTiles; ///< Total number of tiles @@ -169,10 +158,10 @@ int64_t fCurrentRow; ///< current row in memory signed because we need -1 - streamoff fHeapOff; ///< offset from the beginning of the file of the binary data + streamoff fHeapOff; ///< offset from the beginning of the file of the binary data streamoff fHeapFromDataStart; ///< offset from the beginning of the data table - vector>> fCatalog;///< Catalog, i.e. the main table that points to the compressed data. - vector fTileSize; ///< size in bytes of each compressed tile - vector> fTileOffsets; ///< offset from start of tile of a given compressed column + std::vector>> fCatalog; ///< Catalog, i.e. the main table that points to the compressed data. + std::vector fTileSize; ///< size in bytes of each compressed tile + std::vector> fTileOffsets; ///< offset from start of tile of a given compressed column Checksum fRawsum; ///< Checksum of the uncompressed, raw data @@ -183,9 +172,16 @@ uint32_t buffer_size = fTable.bytes_per_row*fNumRowsPerTile; uint32_t compressed_buffer_size = fTable.bytes_per_row*fNumRowsPerTile + - fTable.num_cols*(sizeof(BlockHeader)+256) + //use a bit more memory for block headers. 256 char coding the compression sequence max. - sizeof(TileHeader) + //a bit more for the tile headers - 8;//and a bit more for checksuming - if (buffer_size % 4 != 0) buffer_size += 4 - (buffer_size%4); - if (compressed_buffer_size % 4 != 0) compressed_buffer_size += 4 - (compressed_buffer_size%4); + //use a bit more memory for block headers. 256 char coding the compression sequence max. + fTable.num_cols*(sizeof(FITS::BlockHeader)+256) + + //a bit more for the tile headers + sizeof(FITS::TileHeader) + + //and a bit more for checksuming + 8; + + if (buffer_size % 4 != 0) + buffer_size += 4 - (buffer_size%4); + + if (compressed_buffer_size % 4 != 0) + compressed_buffer_size += 4 - (compressed_buffer_size%4); fBuffer.resize(buffer_size); @@ -198,5 +194,5 @@ void ReadCatalog() { - vector readBuf(16); + std::vector readBuf(16); fCatalog.resize(fNumTiles); @@ -265,5 +261,5 @@ const size_t catSize = fTable.GetHeapShift() + fTable.total_bytes; - vector buf(catSize); + std::vector buf(catSize); read(buf.data(), catSize); @@ -297,8 +293,8 @@ { //read yet another chunk from the file - const int64_t sizeToRead = fTileSize[requestedTile] + sizeof(TileHeader); + const int64_t sizeToRead = fTileSize[requestedTile] + sizeof(FITS::TileHeader); //skip to the beginning of the tile - const int64_t tileStart = fCatalog[requestedTile][0].second - sizeof(TileHeader); + const int64_t tileStart = fCatalog[requestedTile][0].second - sizeof(FITS::TileHeader); seekg(fHeapOff+tileStart); @@ -334,5 +330,5 @@ //uncompress it - UncompressBuffer(requestedTile, thisRoundNumRows, offset+sizeof(TileHeader)); + UncompressBuffer(requestedTile, thisRoundNumRows, offset+sizeof(FITS::TileHeader)); // pointer to column (source buffer) @@ -340,5 +336,5 @@ uint32_t i=0; - for (auto it=fTable.sorted_cols.begin(); it!=fTable.sorted_cols.end(); it++, i++) + for (auto it=fTable.sorted_cols.cbegin(); it!=fTable.sorted_cols.cend(); it++, i++) { char *buffer = fBuffer.data() + it->offset; // pointer to column (destination buffer) @@ -346,32 +342,33 @@ switch (fColumnOrdering[i]) { - case kOrderByRow: - // regular, "semi-transposed" copy - for (char *dest=buffer; destbytes); + src += it->bytes; // next column + } + break; + + case FITS::kOrderByCol: + // transposed copy + for (char *elem=buffer; elembytes; elem+=it->size) // element-by-element (arrays) + { + for (char *dest=elem; destbytes); - src += it->bytes; // next column - } - break; - - case kOrderByCol: - // transposed copy - for (char *elem=buffer; elembytes; elem+=it->size) // element-by-element (arrays) - { - for (char *dest=elem; destsize); src += it->size; // next element - } } + } break; - default: - clear(rdstate()|ios::badbit); - #ifdef __EXCEPTIONS - throw runtime_error("Unkown column ordering scheme found"); - #else - gLog << ___err___ << "ERROR - unkown column ordering scheme" << endl; - return false; - #endif + + default: + clear(rdstate()|ios::badbit); +#ifdef __EXCEPTIONS + throw runtime_error("Unkown column ordering scheme found"); +#else + gLog << ___err___ << "ERROR - unkown column ordering scheme" << endl; + return false; +#endif break; }; @@ -399,5 +396,5 @@ uint32_t numChunks) { - vector uncompressed; + std::vector uncompressed; //read compressed sizes (one per row) @@ -448,12 +445,12 @@ const int64_t compressedOffset = fTileOffsets[catalogCurrentRow][i]+offset; - const BlockHeader* head = reinterpret_cast(&fCompressedBuffer[compressedOffset]); + const FITS::BlockHeader* head = reinterpret_cast(&fCompressedBuffer[compressedOffset]); fColumnOrdering[i] = head->ordering; - const uint32_t numRows = (head->ordering==kOrderByRow) ? thisRoundNumRows : col.num; - const uint32_t numCols = (head->ordering==kOrderByCol) ? thisRoundNumRows : col.num; - - const char *src = fCompressedBuffer.data()+compressedOffset+sizeof(BlockHeader)+sizeof(uint16_t)*head->numProcs; + const uint32_t numRows = (head->ordering==FITS::kOrderByRow) ? thisRoundNumRows : col.num; + const uint32_t numCols = (head->ordering==FITS::kOrderByCol) ? thisRoundNumRows : col.num; + + const char *src = fCompressedBuffer.data()+compressedOffset+sizeof(FITS::BlockHeader)+sizeof(uint16_t)*head->numProcs; for (int32_t j=head->numProcs-1;j >= 0; j--) @@ -463,13 +460,13 @@ switch (head->processings[j]) { - case kFactRaw: + case FITS::kFactRaw: sizeWritten = UncompressUNCOMPRESSED(dest, src, numRows*numCols, col.size); break; - case kFactSmoothing: + case FITS::kFactSmoothing: sizeWritten = UnApplySMOOTHING(reinterpret_cast(dest), numRows*numCols); break; - case kFactHuffman16: + case FITS::kFactHuffman16: sizeWritten = UncompressHUFFMAN16(dest, src, numRows); break; @@ -477,5 +474,6 @@ default: clear(rdstate()|ios::badbit); - ostringstream str; + + std::ostringstream str; str << "Unkown processing applied to data. Col " << i << " proc " << j << " out of " << (int)head->numProcs; #ifdef __EXCEPTIONS @@ -505,13 +503,14 @@ size_t numCols = fTable.num_cols; - vector > > catalog; - - TileHeader tileHead; - BlockHeader columnHead; + std::vector > > catalog; + + FITS::TileHeader tileHead; + FITS::BlockHeader columnHead; + streamoff offsetInHeap = 0; //skip through the heap while (true) { - read((char*)(&tileHead), sizeof(TileHeader)); + read((char*)(&tileHead), sizeof(FITS::TileHeader)); //end of file if (!good()) @@ -526,6 +525,6 @@ //a new tile begins here - catalog.push_back(vector >(0)); - offsetInHeap += sizeof(TileHeader); + catalog.emplace_back(std::vector >(0)); + offsetInHeap += sizeof(FITS::TileHeader); //skip through the columns @@ -535,12 +534,15 @@ if (fTable.sorted_cols[i].num == 0) { - catalog.back().push_back(make_pair(0,0)); + catalog.back().emplace_back(0,0); continue; } + //read column header - read((char*)(&columnHead), sizeof(BlockHeader)); + read((char*)(&columnHead), sizeof(FITS::BlockHeader)); + //corrupted tile if (!good()) break; + catalog.back().emplace_back((int64_t)(columnHead.size),offsetInHeap); offsetInHeap += columnHead.size; @@ -561,5 +563,5 @@ { clear(rdstate()|ios::badbit); - ostringstream str; + std::ostringstream str; str << "Heap data does not agree with header: " << numRows << " calculated vs " << fTable.num_rows << " from header."; #ifdef __EXCEPTIONS