From 4e86123a5af95aed94c28816c2d6f7aa57987468 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 3 Dec 2024 22:10:42 +0000 Subject: [PATCH] feat(): use boost::make_shared b/c most distros can't do allocate_shared for array types. --- tests/counting_allocator.cpp | 168 +++++++++++++++++-------------- utils/common/countingallocator.h | 2 + utils/rowgroup/rowgroup.cpp | 30 +++--- utils/rowgroup/rowgroup.h | 13 +-- 4 files changed, 115 insertions(+), 98 deletions(-) diff --git a/tests/counting_allocator.cpp b/tests/counting_allocator.cpp index 1ca5ccda6..4fd2b71bc 100644 --- a/tests/counting_allocator.cpp +++ b/tests/counting_allocator.cpp @@ -23,117 +23,131 @@ using namespace allocators; // Example class to be managed by the allocator -struct TestClass { - int value; +struct TestClass +{ + int value; - TestClass(int val) : value(val) {} + TestClass(int val) : value(val) + { + } }; -static const constexpr int64_t MemoryAllowance = 10 * 1024 * 1024; +static const constexpr int64_t MemoryAllowance = 10 * 1024 * 1024; // Test Fixture for AtomicCounterAllocator -class CountingAllocatorTest : public ::testing::Test { -protected: - // Atomic counter to track allocated memory - std::atomic allocatedMemory{MemoryAllowance}; +class CountingAllocatorTest : public ::testing::Test +{ + protected: + // Atomic counter to track allocated memory + std::atomic allocatedMemory{MemoryAllowance}; - // Custom allocator instance - CountingAllocator allocator; + // Custom allocator instance + CountingAllocator allocator; - // Constructor - CountingAllocatorTest() - : allocatedMemory(MemoryAllowance), allocator(allocatedMemory, MemoryAllowance / 100) {} + // Constructor + CountingAllocatorTest() + : allocatedMemory(MemoryAllowance), allocator(allocatedMemory, MemoryAllowance / 100) + { + } - // Destructor - ~CountingAllocatorTest() override = default; + // Destructor + ~CountingAllocatorTest() override = default; }; // Test 1: Allocation increases the counter correctly -TEST_F(CountingAllocatorTest, Allocation) { - const std::size_t numObjects = 5; - TestClass* ptr = allocator.allocate(numObjects); - EXPECT_NE(ptr, nullptr); - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance - numObjects * static_cast(sizeof(TestClass))); - allocator.deallocate(ptr, numObjects); +TEST_F(CountingAllocatorTest, Allocation) +{ + const std::size_t numObjects = 5; + TestClass* ptr = allocator.allocate(numObjects); + EXPECT_NE(ptr, nullptr); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance - numObjects * static_cast(sizeof(TestClass))); + allocator.deallocate(ptr, numObjects); } // Test 2: Deallocation decreases the counter correctly -TEST_F(CountingAllocatorTest, Deallocation) { - const std::size_t numObjects = 3; - TestClass* ptr = allocator.allocate(numObjects); - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance - numObjects * static_cast(sizeof(TestClass))); +TEST_F(CountingAllocatorTest, Deallocation) +{ + const std::size_t numObjects = 3; + TestClass* ptr = allocator.allocate(numObjects); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance - numObjects * static_cast(sizeof(TestClass))); - allocator.deallocate(ptr, numObjects); - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); + allocator.deallocate(ptr, numObjects); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); } // Test 3: Allocator equality based on shared counter -TEST_F(CountingAllocatorTest, AllocatorEquality) { - CountingAllocator allocator1(allocatedMemory); - CountingAllocator allocator2(allocatedMemory); - EXPECT_TRUE(allocator1 == allocator2); +TEST_F(CountingAllocatorTest, AllocatorEquality) +{ + CountingAllocator allocator1(allocatedMemory); + CountingAllocator allocator2(allocatedMemory); + EXPECT_TRUE(allocator1 == allocator2); - std::atomic anotherCounter(0); - CountingAllocator allocator3(anotherCounter); - EXPECT_FALSE(allocator1 == allocator3); + std::atomic anotherCounter(0); + CountingAllocator allocator3(anotherCounter); + EXPECT_FALSE(allocator1 == allocator3); } // Test 4: Using allocator with std::allocate_shared -TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator) { - // Create a shared_ptr using allocate_shared with the custom allocator - std::shared_ptr ptr = std::allocate_shared(allocator, 100); +TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator) +{ + // Create a shared_ptr using allocate_shared with the custom allocator + std::shared_ptr ptr = std::allocate_shared(allocator, 100); - // Check that the counter has increased by the size of TestClass plus control block - // Exact size depends on the implementation, so we verify it's at least sizeof(TestClass) - EXPECT_LE(allocatedMemory.load(), MemoryAllowance - static_cast(sizeof(TestClass))); + // Check that the counter has increased by the size of TestClass plus control block + // Exact size depends on the implementation, so we verify it's at least sizeof(TestClass) + EXPECT_LE(allocatedMemory.load(), MemoryAllowance - static_cast(sizeof(TestClass))); - // Reset the shared_ptr and check that the counter decreases appropriately - ptr.reset(); - // After deallocation, the counter should return to zero - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); + // Reset the shared_ptr and check that the counter decreases appropriately + ptr.reset(); + // After deallocation, the counter should return to zero + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); - auto deleter = [this](TestClass* ptr) { - this->allocator.deallocate(ptr, 1); - }; - ptr.reset(allocator.allocate(1), deleter); - EXPECT_LE(allocatedMemory.load(), MemoryAllowance - static_cast(sizeof(TestClass))); + auto deleter = [this](TestClass* ptr) { this->allocator.deallocate(ptr, 1); }; + ptr.reset(allocator.allocate(1), deleter); + EXPECT_LE(allocatedMemory.load(), MemoryAllowance - static_cast(sizeof(TestClass))); - ptr.reset(); - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); + ptr.reset(); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); } // Test 5: Thread Safety - Concurrent Allocations and Deallocations -TEST_F(CountingAllocatorTest, ThreadSafety) { - const std::size_t numThreads = 100; - const std::size_t allocationsPerThread = 3; +TEST_F(CountingAllocatorTest, ThreadSafety) +{ + const std::size_t numThreads = 100; + const std::size_t allocationsPerThread = 3; - auto worker = [this]() { - for (std::size_t i = 0; i < allocationsPerThread; ++i) { - TestClass* ptr = allocator.allocate(1); - allocator.deallocate(ptr, 1); - } - }; - - std::vector threads; - // Launch multiple threads performing allocations and deallocations - for (std::size_t i = 0; i < numThreads; ++i) { - threads.emplace_back(worker); + auto worker = [this]() + { + for (std::size_t i = 0; i < allocationsPerThread; ++i) + { + TestClass* ptr = allocator.allocate(1); + allocator.deallocate(ptr, 1); } + }; - // Wait for all threads to finish - for (auto& th : threads) { - th.join(); - } + std::vector threads; + // Launch multiple threads performing allocations and deallocations + for (std::size_t i = 0; i < numThreads; ++i) + { + threads.emplace_back(worker); + } - // After all allocations and deallocations, the counter should be zero - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); + // Wait for all threads to finish + for (auto& th : threads) + { + th.join(); + } + + // After all allocations and deallocations, the counter should be zero + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); } // Test 6: Allocating zero objects should not change the counter -TEST_F(CountingAllocatorTest, AllocateZeroObjects) { - TestClass* ptr = allocator.allocate(0); - EXPECT_NE(ptr, nullptr); - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); - allocator.deallocate(ptr, 0); - EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); +TEST_F(CountingAllocatorTest, AllocateZeroObjects) +{ + TestClass* ptr = allocator.allocate(0); + EXPECT_NE(ptr, nullptr); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); + allocator.deallocate(ptr, 0); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); } \ No newline at end of file diff --git a/utils/common/countingallocator.h b/utils/common/countingallocator.h index 82e8f9105..ea23d41ba 100644 --- a/utils/common/countingallocator.h +++ b/utils/common/countingallocator.h @@ -61,6 +61,7 @@ public: T* ptr = static_cast(::operator new(n * sizeof(T))); // std::cout << "[Allocate] " << n * sizeof(T) << " bytes at " << static_cast(ptr) // << ". current timit: " << std::dec << memoryLimitRef_.load() << std::hex << " bytes.\n"; + // std::cout << std::dec; return ptr; } @@ -87,6 +88,7 @@ public: memoryLimitRef_.fetch_add(n * sizeof(T), std::memory_order_relaxed); // std::cout << "[Deallocate] " << n * sizeof(T) << " bytes from " << static_cast(ptr) // << ". current timit: " << std::dec << memoryLimitRef_.load() << std::hex << " bytes.\n"; + // std::cout << std::dec; } // Equality operators (allocators are equal if they share the same counter) diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 2dd7b190f..20aba90ae 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -100,12 +100,12 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) if ((len + 4) >= CHUNK_SIZE) { auto allocSize = len + sizeof(MemChunk) + 4; - if (alloc) - { - cout << "StringStore::storeString longStrings with alloc " << std::endl; - longStrings.emplace_back(std::allocate_shared(*alloc, allocSize)); - } - else + // if (alloc) + // { + // cout << "StringStore::storeString longStrings with alloc " << std::endl; + // longStrings.emplace_back(std::allocate_shared(*alloc, allocSize)); + // } + // else { cout << "StringStore::storeString longStrings no alloc " << std::endl; longStrings.emplace_back(std::make_shared(allocSize)); @@ -128,14 +128,14 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) if (alloc) { cout << "StringStore::storeString with alloc " << std::endl; - mem.emplace_back(std::allocate_shared(*alloc, CHUNK_SIZE + sizeof(MemChunk))); - // std::allocate_shared) newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); + mem.emplace_back(boost::allocate_shared(*alloc, CHUNK_SIZE + sizeof(MemChunk))); + // boost::allocate_shared) newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); } else { cout << "StringStore::storeString no alloc " << std::endl; - mem.emplace_back(std::make_shared(CHUNK_SIZE + sizeof(MemChunk))); - // mem.emplace_back(std::allocate_shared(*alloc, CHUNK_SIZE + sizeof(MemChunk))); + mem.emplace_back(boost::make_shared(CHUNK_SIZE + sizeof(MemChunk))); + // mem.emplace_back(boost::allocate_shared(*alloc, CHUNK_SIZE + sizeof(MemChunk))); // std::shared_ptr newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); } // mem.push_back(newOne); @@ -209,12 +209,12 @@ void StringStore::deserialize(ByteStream& bs) if (alloc) { cout << "StringStore::deserialize with alloc " << std::endl; - mem.emplace_back(std::allocate_shared(*alloc, size + sizeof(MemChunk))); + mem.emplace_back(boost::allocate_shared(*alloc, size + sizeof(MemChunk))); } else { cout << "StringStore::deserialize no alloc " << std::endl; - mem.emplace_back(std::make_shared(size + sizeof(MemChunk))); + mem.emplace_back(boost::make_shared(size + sizeof(MemChunk))); } // mem[i].reset(new uint8_t[size + sizeof(MemChunk)]); mc = (MemChunk*)mem[i].get(); @@ -230,7 +230,7 @@ void StringStore::deserialize(ByteStream& bs) void StringStore::clear() { - vector > emptyv; + vector > emptyv; vector > emptyv2; mem.swap(emptyv); longStrings.swap(emptyv2); @@ -367,7 +367,7 @@ RGData::RGData(const RowGroup& rg, allocators::CountingAllocator* { // rowData = shared_ptr(buf, [alloc, allocSize](uint8_t* p) { alloc->deallocate(p, allocSize); // }); - rowData = std::allocate_shared(*alloc, rg.getMaxDataSize()); + rowData = boost::allocate_shared(*alloc, rg.getMaxDataSize()); // rowData = std::make_shared(uint8_t[rg.getMaxDataSize()]); if (rg.usesStringTable()) @@ -381,7 +381,7 @@ void RGData::reinit(const RowGroup& rg, uint32_t rowCount) if (alloc) { cout << "RGData::reinit with alloc " << std::endl; - rowData = std::allocate_shared(*alloc, rg.getDataSize(rowCount)); + rowData = boost::allocate_shared(*alloc, rg.getDataSize(rowCount)); } else { diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e5dd93f02..665792f9d 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -190,7 +190,7 @@ class StringStore std::string empty_str; static constexpr const uint32_t CHUNK_SIZE = 64 * 1024; // allocators like powers of 2 - std::vector> mem; + std::vector> mem; // To store strings > 64KB (BLOB/TEXT) std::vector> longStrings; @@ -286,7 +286,7 @@ class RGData void clear(); void reinit(const RowGroup& rg); void reinit(const RowGroup& rg, uint32_t rowCount); - inline void setStringStore(std::shared_ptr& ss) + inline void setStringStore(boost::shared_ptr& ss) { strings = ss; } @@ -327,8 +327,8 @@ class RGData private: uint32_t rowSize = 0; // can't be. uint32_t columnCount = 0; // shouldn't be, but... - std::shared_ptr rowData; - std::shared_ptr strings; + boost::shared_ptr rowData; + boost::shared_ptr strings; std::shared_ptr userDataStore; allocators::CountingAllocator* alloc = nullptr; @@ -1599,7 +1599,7 @@ class RowGroup : public messageqcpp::Serializeable const uint16_t& blockNum); inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum); - inline void setStringStore(std::shared_ptr); + inline void setStringStore(boost::shared_ptr); const CHARSET_INFO* getCharset(uint32_t col); @@ -1911,7 +1911,8 @@ inline uint32_t RowGroup::getStringTableThreshold() const return sTableThreshold; } -inline void RowGroup::setStringStore(std::shared_ptr ss) +// WIP mb unused +inline void RowGroup::setStringStore(boost::shared_ptr ss) { if (useStringTable) {