Index: /trunk/Mars/mcore/zofits.h =================================================================== --- /trunk/Mars/mcore/zofits.h (revision 17242) +++ /trunk/Mars/mcore/zofits.h (revision 17243) @@ -1,2 +1,5 @@ +#ifndef FACT_zofits +#define FACT_zofits + /* * zofits.h @@ -57,16 +60,22 @@ public: + /// static setter for the default number of threads to use. -1 means all available physical cores + static uint32_t DefaultNumThreads(const uint32_t &_n=-2) { static uint32_t n=0; if (int32_t(_n)<-1) n=_n; return n; } + static uint32_t DefaultMaxMemory(const uint32_t &_n=0) { static uint32_t n=1000000; if (_n>0) n=_n; return n; } + static uint32_t DefaultMaxNumTiles(const uint32_t &_n=0) { static uint32_t n=1000; if (_n>0) n=_n; return n; } + static uint32_t DefaultNumRowsPerTile(const uint32_t &_n=0) { static uint32_t n=100; if (_n>0) n=_n; return n; } + /// constructors /// @param numTiles how many data groups should be pre-reserved ? /// @param rowPerTile how many rows will be grouped together in a single tile /// @param maxUsableMem how many bytes of memory can be used by the compression buffers - zofits(uint32_t numTiles = fgNumTiles, - uint32_t rowPerTile = fgRowPerTile, - uint64_t maxUsableMem= fgMaxUsableMem) : ofits(), - fMemPool(0, maxUsableMem), + zofits(uint32_t numTiles = DefaultMaxNumTiles(), + uint32_t rowPerTile = DefaultNumRowsPerTile(), + uint32_t maxUsableMem= DefaultMaxMemory()) : ofits(), + fMemPool(0, maxUsableMem*1000), fWriteToDiskQueue(bind(&zofits::WriteBufferToDisk, this, placeholders::_1), false) { - InitMemberVariables(numTiles, rowPerTile, maxUsableMem); - SetNumThreads(fgNumQueues); + InitMemberVariables(numTiles, rowPerTile, maxUsableMem*1000); + SetNumThreads(DefaultNumThreads()); } @@ -76,17 +85,12 @@ /// @param maxUsableMem how many bytes of memory can be used by the compression buffers zofits(const char* fname, - uint32_t numTiles = fgNumTiles, - uint32_t rowPerTile = fgRowPerTile, - uint64_t maxUsableMem= fgMaxUsableMem) : ofits(fname), - fMemPool(0, maxUsableMem), + uint32_t numTiles = DefaultMaxNumTiles(), + uint32_t rowPerTile = DefaultNumRowsPerTile(), + uint32_t maxUsableMem= DefaultMaxMemory()) : ofits(fname), + fMemPool(0, maxUsableMem*1000), fWriteToDiskQueue(bind(&zofits::WriteBufferToDisk, this, placeholders::_1), false) { - InitMemberVariables(numTiles, rowPerTile, maxUsableMem); - SetNumThreads(fgNumQueues); - } - - /// destructors - virtual ~zofits() - { + InitMemberVariables(numTiles, rowPerTile, maxUsableMem*1000); + SetNumThreads(DefaultNumThreads()); } @@ -577,18 +581,7 @@ } - /// static setter for the default number of threads to use. -1 means all available physical cores - static void SetDefaultNumThreads (int32_t num) { fgNumQueues = num;} - static void SetDefaultNumTiles (uint32_t num) { fgNumTiles = num;} - static void SetDefaultNumRowPerTile(uint32_t num) { fgRowPerTile = num;} - static void SetDefaultMaxUsableMem (uint64_t size) { fgMaxUsableMem = size;} - - static int32_t GetDefaultNumThreads() { return fgNumQueues;} - static uint32_t GetDefaultNumTiles() { return fgNumTiles;} - static uint32_t GetDefaultNumRowPerTile() { return fgRowPerTile;} - static uint64_t GetDefaulMaxUsableMem() { return fgMaxUsableMem;} - /// Get and set the actual number of threads for this object int32_t GetNumThreads() const { return fNumQueues;} - bool SetNumThreads(int32_t num) + bool SetNumThreads(uint32_t num) { if (is_open()) @@ -604,46 +597,21 @@ //get number of physically available threads #ifdef USE_BOOST_THREADS - int32_t num_available_cores = boost::thread::hardware_concurrency(); + unsigned int num_available_cores = boost::thread::hardware_concurrency(); #else - int32_t num_available_cores = thread::hardware_concurrency(); -#endif - + unsigned int num_available_cores = thread::hardware_concurrency(); +#endif // could not detect number of available cores from system properties... - // assume that 5 cores are available (4 compression, 1 write) if (num_available_cores == 0) - num_available_cores = 5; - - // Throw an exception if too many cores are requested + num_available_cores = 1; + + // leave one core for the main thread and one for the writing if (num > num_available_cores) - { - ostringstream str; - str << "You will be using more threads(" << num << ") than available cores(" << num_available_cores << "). Expect sub-optimal performances"; -#ifdef __EXCEPTIONS - throw runtime_error(str.str()); -#else - gLog << ___err___ << "WARNING - " << str.str() << endl; -#endif - } - - if (num == -1) - num = num_available_cores-2; // 1 for writing, 1 for the main thread - - if (fCompressionQueues.size() == (uint32_t)num) - return true; - - //cannot be const, as resize does not want it that way - Queue queue(bind(&zofits::CompressBuffer, this, placeholders::_1), false); - - //shrink if required - if ((uint32_t)num < fCompressionQueues.size()) - { - fCompressionQueues.resize(num, queue); - return true; - } - - //grow if required - fCompressionQueues.resize(num, queue); - - fNumQueues = num; + num = num_available_cores>2 ? num_available_cores-2 : 1; + + if (fCompressionQueues.size() != uint32_t(num)) + { + fCompressionQueues.resize(num, Queue(bind(&zofits::CompressBuffer, this, placeholders::_1), false)); + fNumQueues = num; + } return true; @@ -991,12 +959,8 @@ //thread related stuff - MemoryManager fMemPool; ///< Actual memory manager, providing memory for the compression buffers - static int32_t fgNumQueues; ///< Default number of threads to be used by the objects - static uint32_t fgNumTiles; ///< Default number of reserved tiles - static uint32_t fgRowPerTile; ///< Default number of rows per tile - static uint64_t fgMaxUsableMem; ///< Default usable memory PER OBJECT - int32_t fNumQueues; ///< Current number of threads that will be used by this object - uint64_t fMaxUsableMem; ///< Maximum number of bytes that can be allocated by the memory manager - int32_t fLatestWrittenTile; ///< Index of the last tile written to disk (for correct ordering while using several threads) + MemoryManager fMemPool; ///< Actual memory manager, providing memory for the compression buffers + int32_t fNumQueues; ///< Current number of threads that will be used by this object + uint64_t fMaxUsableMem; ///< Maximum number of bytes that can be allocated by the memory manager + int32_t fLatestWrittenTile; ///< Index of the last tile written to disk (for correct ordering while using several threads) vector> fCompressionQueues; ///< Processing queues (=threads) @@ -1047,10 +1011,7 @@ }; -int32_t zofits::fgNumQueues = 0; -uint32_t zofits::fgNumTiles = 1000; -uint32_t zofits::fgRowPerTile = 100; -uint64_t zofits::fgMaxUsableMem = 1073741824; // one gigabyte - #ifndef __MARS__ }; //namespace std #endif + +#endif