Changeset 16443 for trunk/Mars/mcore
- Timestamp:
- 05/29/13 22:02:43 (11 years ago)
- Location:
- trunk/Mars/mcore
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Mars/mcore/factfits.h
r16418 r16443 21 21 // Default constructor 22 22 factfits(const string& fname, const string& tableName="", bool force=false) : 23 zfits(fname, tableName, force) 23 zfits(fname, tableName, force), 24 fOffsetCalibration(0), 25 fOffsetStartCellData(0), 26 fOffsetData(0), 27 fNumRoi(0) 24 28 { 25 29 if (init()) … … 29 33 // Alternative constructor 30 34 factfits(const string& fname, const string& fout, const string& tableName, bool force=false) : 31 zfits(fname, fout, tableName, force) 35 zfits(fname, fout, tableName, force), 36 fOffsetCalibration(0), 37 fOffsetStartCellData(0), 38 fOffsetData(0), 39 fNumRoi(0) 32 40 { 33 41 if (init()) … … 35 43 } 36 44 37 // 38 #if !defined(__MARS__) && !defined(__CINT__) 39 bool GetRow(size_t row, bool check=true) 40 #else 41 bool GetRowNum(size_t row, bool check=true) 42 #endif 45 private: 46 47 void StageRow(size_t row, char* dest) 43 48 { 44 #if !defined(__MARS__) && !defined(__CINT__) 45 if (!zfits::GetRow(row, check)) 46 return false; 47 #else 48 if (!zfits::GetRowNum(row, check)) 49 return false; 50 #endif 49 zfits::StageRow(row, dest); 51 50 // This file does not contain fact data or no calibration to be applied 52 if (fOffsetCalibration.empty())53 return true;51 if (fOffsetCalibration.empty()) 52 return; 54 53 55 //re-get the pointer to the data to access the offsets56 const uint8_t offset = (row*fTable.bytes_per_row)%4;54 //re-get the pointer to the data to access the offsets 55 const uint8_t offset = (row*fTable.bytes_per_row)%4; 57 56 58 int16_t *startCell = reinterpret_cast<int16_t*>(fBufferRow.data() + offset + fOffsetStartCellData); 57 int16_t *startCell = reinterpret_cast<int16_t*>(fBufferRow.data() + offset + fOffsetStartCellData); 58 int16_t *data = reinterpret_cast<int16_t*>(fBufferRow.data() + offset + fOffsetData); 59 59 60 const Pointers::iterator dtaIt = fPointers.find("Data"); 61 if (dtaIt == fPointers.end()) return true; 62 int16_t *data = reinterpret_cast<int16_t*>(dtaIt->second); 63 64 for (uint32_t i=0; i<1440*1024; i+=1024, startCell++) 65 { 66 for (uint32_t j=0; j<fNumRoi; j++, data++) 67 *data += fOffsetCalibration[i + (*startCell+j)%1024]; 68 } 69 70 return true; 60 for (uint32_t i=0; i<1440*1024; i+=1024, startCell++) 61 { 62 if ((*startCell) > -1) 63 for (uint32_t j=0; j<fNumRoi; j++, data++) 64 *data += fOffsetCalibration[i + (*startCell+j)%1024]; 65 } 71 66 } 72 73 private:74 67 75 68 bool init() … … 104 97 void readDrsCalib(const string& fileName) 105 98 { 106 zfits calib(fileName, "DrsCalib"); 99 //should not be mandatory, but improves the perfs a lot when reading not compressed, gzipped files 100 if (!IsCompressedFITS()) 101 return; 102 103 zfits calib(fileName, "IntCalibration"); 104 107 105 if (calib.bad()) 108 106 { … … 119 117 clear(rdstate()|ios::badbit); 120 118 #ifdef __EXCEPTIONS 121 throw runtime_error("Table ' DrsCalib' found, but not with one row as expected");119 throw runtime_error("Table 'IntCalibration' found, but not with one row as expected"); 122 120 #else 123 gLog << ___err___ << "ERROR - Table ' DrsCalib' found, but not with one row as expected" << endl;121 gLog << ___err___ << "ERROR - Table 'IntCalibration' found, but not with one row as expected" << endl; 124 122 return; 125 123 #endif … … 164 162 vector<int16_t> fOffsetCalibration; ///< integer values of the drs calibration used for compression 165 163 164 size_t fOffsetStartCellData; 166 165 size_t fOffsetData; 167 size_t fOffsetStartCellData;168 166 169 167 uint16_t fNumRoi; -
trunk/Mars/mcore/fits.h
r16426 r16443 612 612 peek(); 613 613 if (eof() && !bad() && !tableName.empty()) 614 { 615 cout << "END OF FILE !" << endl; 614 616 break; 615 617 } 616 618 // FIXME: Set limit on memory consumption 617 619 const int rc = ReadBlock(block); … … 741 743 } 742 744 745 virtual void WriteRowToCopyFile(size_t row) 746 { 747 if (row==fRow+1 && !fTable.isCompressed) 748 { 749 const uint8_t offset = (row*fTable.bytes_per_row)%4; 750 751 fChkData.add(fBufferRow); 752 if (fCopy.is_open() && fCopy.good()) 753 fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row); 754 if (!fCopy) 755 clear(rdstate()|ios::badbit); 756 } 757 else 758 if (fCopy.is_open()) 759 clear(rdstate()|ios::badbit); 760 } 743 761 uint8_t ReadRow(size_t row) 744 762 { … … 764 782 StageRow(row, fBufferRow.data()+offset); 765 783 766 if (row==fRow+1) 767 { 768 fChkData.add(fBufferRow); 769 if (fCopy.is_open() && fCopy.good()) 770 fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row); 771 if (!fCopy) 772 clear(rdstate()|ios::badbit); 773 } 774 else 775 if (fCopy.is_open()) 776 clear(rdstate()|ios::badbit); 784 WriteRowToCopyFile(row); 777 785 778 786 fRow = row; -
trunk/Mars/mcore/zfits.h
r16426 r16443 33 33 fNumRowsPerTile(0), 34 34 fCurrentRow(-1), 35 fHeapOff(0) 35 fHeapOff(0), 36 fTileSize(0) 36 37 { 37 38 InitCompressionReading(); … … 47 48 fNumRowsPerTile(0), 48 49 fCurrentRow(-1), 49 fHeapOff(0) 50 fHeapOff(0), 51 fTileSize(0) 50 52 { 51 53 InitCompressionReading(); … … 60 62 fRow++; 61 63 return true; 64 } 65 protected: 66 67 // Stage the requested row to internal buffer 68 // Does NOT return data to users 69 virtual void StageRow(size_t row, char* dest) 70 { 71 if (!fTable.isCompressed) 72 { 73 fits::StageRow(row, dest); 74 return; 75 } 76 77 ReadBinaryRow(row, dest); 62 78 } 63 79 … … 82 98 } 83 99 84 // Stage the requested row to internal buffer85 // Does NOT return data to users86 void StageRow(size_t row, char* dest)87 {88 if (!fTable.isCompressed)89 {90 fits::StageRow(row, dest);91 return;92 }93 94 ReadBinaryRow(row, dest);95 }96 97 100 // Copy decompressed data to location requested by user 98 101 void MoveColumnDataToUserSpace(char* dest, const char* src, const Table::Column& c) … … 118 121 119 122 vector<vector<pair<int64_t, int64_t> > > fCatalog;///< Catalog, i.e. the main table that points to the compressed data. 123 vector<size_t> fTileSize; ///< size in bytes of each compressed tile 124 vector<vector<size_t> > fTileOffsets; ///< offset from start of tile of a given compressed column 120 125 121 126 void AllocateBuffers() … … 127 132 128 133 fTransposedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile); 129 fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + 1024*1024); //use a bit more memory, in case the compression algorithms uses more134 fCompressedBuffer.resize(fTable.bytes_per_row*fNumRowsPerTile + fTable.num_cols); //use a bit more memory for compression flags 130 135 } 131 136 … … 139 144 fCatalog.resize(fNumTiles); 140 145 146 const streampos catalogStart = tellg(); 147 148 //do the actual reading 141 149 for (uint32_t i=0;i<fNumTiles;i++) 142 150 for (uint32_t j=0;j<fTable.num_cols;j++) … … 147 155 int64_t tempValues[2] = {0,0}; 148 156 revcpy<8>(reinterpret_cast<char*>(tempValues), readBuf, 2); 149 157 if (tempValues[0] < 0 || tempValues[1] < 0) 158 { 159 #ifdef __EXCEPTIONS 160 throw runtime_error("ERROR: negative value in the catalog"); 161 #else 162 gLog << ___err ___ << "ERROR: negative value in the catalog" << endl; 163 return; 164 #endif 165 } 150 166 //add catalog entry 151 167 fCatalog[i].emplace_back(tempValues[0], tempValues[1]); 152 168 } 153 169 170 //compute the total size of each compressed tile 171 fTileSize.resize(fNumTiles); 172 fTileOffsets.resize(fNumTiles); 173 for (uint32_t i=0;i<fNumTiles;i++) 174 { 175 fTileSize[i] = 0; 176 for (uint32_t j=0;j<fTable.num_cols;j++) 177 { 178 fTileSize[i] += fCatalog[i][j].first; 179 fTileOffsets[i].emplace_back(fCatalog[i][j].second - fCatalog[i][0].second); 180 } 181 } 154 182 //see if there is a gap before heap data 155 183 fHeapOff = tellg()+fTable.GetHeapShift(); 156 } 157 184 185 if (!fCopy.is_open()) 186 return; 187 188 //write catalog and heap gap to target file 189 seekg(catalogStart); 190 191 const size_t catSize = fTable.GetHeapShift() + fTable.total_bytes; 192 193 vector<char> buf(catSize); 194 read(buf.data(), catSize); 195 196 fCopy.write(buf.data(), catSize); 197 if (!fCopy) 198 clear(rdstate()|ios::badbit); 199 } 200 //overrides fits.h method with empty one 201 //work is done in ReadBinaryRow because it requires volatile data from ReadBinaryRow 202 virtual void WriteRowToCopyFile(size_t ) 203 { 204 205 } 158 206 // Compressed versin of the read row 159 207 bool ReadBinaryRow(const size_t &rowNum, char *bufferToRead) … … 164 212 const uint32_t requestedTile = rowNum/fNumRowsPerTile; 165 213 const uint32_t currentTile = fCurrentRow/fNumRowsPerTile; 214 const size_t previousRow = fCurrentRow; 166 215 167 216 fCurrentRow = rowNum; … … 171 220 { 172 221 //read yet another chunk from the file 173 //the size that we should read is in the catalog. we should sum up the sizes of all columns 174 const uint32_t currentCatRow = fCurrentRow/fNumRowsPerTile; 175 176 int64_t sizeToRead = 0; 177 for (uint32_t i=0;i<fCatalog[currentCatRow].size();i++) 178 sizeToRead += fCatalog[currentCatRow][i].first; 222 const int64_t sizeToRead = fTileSize[requestedTile]; 179 223 180 224 //skip to the beginning of the tile 181 seekg(fHeapOff+fCatalog[ currentCatRow][0].second);225 seekg(fHeapOff+fCatalog[requestedTile][0].second); 182 226 read(fCompressedBuffer.data(), sizeToRead); 183 227 228 if (fCurrentRow == previousRow+1 && 229 fCopy.is_open() && 230 fCopy.good()) 231 { 232 fCopy.write(fCompressedBuffer.data(), sizeToRead); 233 if (!fCopy) 234 clear(rdstate()|ios::badbit); 235 } 236 else 237 if (fCopy.is_open()) 238 clear(rdstate()|ios::badbit); 239 184 240 const uint32_t thisRoundNumRows = (GetNumRows()<fCurrentRow + fNumRowsPerTile) ? GetNumRows()%fNumRowsPerTile : fNumRowsPerTile; 185 241 186 242 //uncompress it 187 UncompressBuffer( currentCatRow, thisRoundNumRows);243 UncompressBuffer(requestedTile, thisRoundNumRows); 188 244 189 245 // pointer to column (source buffer) … … 301 357 302 358 //get the compression flag 303 const int64_t compressedOffset = f Catalog[catalogCurrentRow][i].second - fCatalog[catalogCurrentRow][0].second;359 const int64_t compressedOffset = fTileOffsets[catalogCurrentRow][i];//fCatalog[catalogCurrentRow][i].second - fCatalog[catalogCurrentRow][0].second; 304 360 const char compressedFlag = fCompressedBuffer[compressedOffset]; 305 361
Note:
See TracChangeset
for help on using the changeset viewer.