database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

feat(): use boost::make_shared b/c most distros can't do allocate_shared for array types.

Browse Source
This commit is contained in:
drrtuy
2024-12-03 22:10:42 +00:00
parent 5f1bd3be12
commit 4e86123a5a
4 changed files with 115 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
tests/counting_allocator.cpp
View File

@@ -23,17 +23,21 @@
using namespace allocators; using namespace allocators;
// Example class to be managed by the allocator // Example class to be managed by the allocator
struct TestClass { struct TestClass
{
int value; int value;
TestClass(int val) : value(val) {} TestClass(int val) : value(val)
{
}
}; };
static const constexpr int64_t MemoryAllowance = 10 * 1024 * 1024; static const constexpr int64_t MemoryAllowance = 10 * 1024 * 1024;
// Test Fixture for AtomicCounterAllocator // Test Fixture for AtomicCounterAllocator
class CountingAllocatorTest : public ::testing::Test { class CountingAllocatorTest : public ::testing::Test
protected: {
protected:
// Atomic counter to track allocated memory // Atomic counter to track allocated memory
std::atomic<int64_t> allocatedMemory{MemoryAllowance}; std::atomic<int64_t> allocatedMemory{MemoryAllowance};
@@ -42,14 +46,17 @@ protected:
// Constructor // Constructor
CountingAllocatorTest() CountingAllocatorTest()
: allocatedMemory(MemoryAllowance), allocator(allocatedMemory, MemoryAllowance / 100) {} : allocatedMemory(MemoryAllowance), allocator(allocatedMemory, MemoryAllowance / 100)
{
}
// Destructor // Destructor
~CountingAllocatorTest() override = default; ~CountingAllocatorTest() override = default;
}; };
// Test 1: Allocation increases the counter correctly // Test 1: Allocation increases the counter correctly
TEST_F(CountingAllocatorTest, Allocation) { TEST_F(CountingAllocatorTest, Allocation)
{
const std::size_t numObjects = 5; const std::size_t numObjects = 5;
TestClass* ptr = allocator.allocate(numObjects); TestClass* ptr = allocator.allocate(numObjects);
EXPECT_NE(ptr, nullptr); EXPECT_NE(ptr, nullptr);
@@ -58,7 +65,8 @@ TEST_F(CountingAllocatorTest, Allocation) {
} }
// Test 2: Deallocation decreases the counter correctly // Test 2: Deallocation decreases the counter correctly
TEST_F(CountingAllocatorTest, Deallocation) { TEST_F(CountingAllocatorTest, Deallocation)
{
const std::size_t numObjects = 3; const std::size_t numObjects = 3;
TestClass* ptr = allocator.allocate(numObjects); TestClass* ptr = allocator.allocate(numObjects);
EXPECT_EQ(allocatedMemory.load(), MemoryAllowance - numObjects * static_cast<int64_t>(sizeof(TestClass))); EXPECT_EQ(allocatedMemory.load(), MemoryAllowance - numObjects * static_cast<int64_t>(sizeof(TestClass)));
@@ -68,7 +76,8 @@ TEST_F(CountingAllocatorTest, Deallocation) {
} }
// Test 3: Allocator equality based on shared counter // Test 3: Allocator equality based on shared counter
TEST_F(CountingAllocatorTest, AllocatorEquality) { TEST_F(CountingAllocatorTest, AllocatorEquality)
{
CountingAllocator<TestClass> allocator1(allocatedMemory); CountingAllocator<TestClass> allocator1(allocatedMemory);
CountingAllocator<TestClass> allocator2(allocatedMemory); CountingAllocator<TestClass> allocator2(allocatedMemory);
EXPECT_TRUE(allocator1 == allocator2); EXPECT_TRUE(allocator1 == allocator2);
@@ -79,7 +88,8 @@ TEST_F(CountingAllocatorTest, AllocatorEquality) {
} }
// Test 4: Using allocator with std::allocate_shared // Test 4: Using allocator with std::allocate_shared
TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator) { TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator)
{
// Create a shared_ptr using allocate_shared with the custom allocator // Create a shared_ptr using allocate_shared with the custom allocator
std::shared_ptr<TestClass> ptr = std::allocate_shared<TestClass>(allocator, 100); std::shared_ptr<TestClass> ptr = std::allocate_shared<TestClass>(allocator, 100);
@@ -92,9 +102,7 @@ TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator) {
// After deallocation, the counter should return to zero // After deallocation, the counter should return to zero
EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); EXPECT_EQ(allocatedMemory.load(), MemoryAllowance);
auto deleter = [this](TestClass* ptr) { auto deleter = [this](TestClass* ptr) { this->allocator.deallocate(ptr, 1); };
this->allocator.deallocate(ptr, 1);
};
ptr.reset(allocator.allocate(1), deleter); ptr.reset(allocator.allocate(1), deleter);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - static_cast<int64_t>(sizeof(TestClass))); EXPECT_LE(allocatedMemory.load(), MemoryAllowance - static_cast<int64_t>(sizeof(TestClass)));
@@ -103,12 +111,15 @@ TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator) {
} }
// Test 5: Thread Safety - Concurrent Allocations and Deallocations // Test 5: Thread Safety - Concurrent Allocations and Deallocations
TEST_F(CountingAllocatorTest, ThreadSafety) { TEST_F(CountingAllocatorTest, ThreadSafety)
{
const std::size_t numThreads = 100; const std::size_t numThreads = 100;
const std::size_t allocationsPerThread = 3; const std::size_t allocationsPerThread = 3;
auto worker = [this]() { auto worker = [this]()
for (std::size_t i = 0; i < allocationsPerThread; ++i) { {
for (std::size_t i = 0; i < allocationsPerThread; ++i)
{
TestClass* ptr = allocator.allocate(1); TestClass* ptr = allocator.allocate(1);
allocator.deallocate(ptr, 1); allocator.deallocate(ptr, 1);
} }
@@ -116,12 +127,14 @@ TEST_F(CountingAllocatorTest, ThreadSafety) {
std::vector<std::thread> threads; std::vector<std::thread> threads;
// Launch multiple threads performing allocations and deallocations // Launch multiple threads performing allocations and deallocations
for (std::size_t i = 0; i < numThreads; ++i) { for (std::size_t i = 0; i < numThreads; ++i)
{
threads.emplace_back(worker); threads.emplace_back(worker);
} }
// Wait for all threads to finish // Wait for all threads to finish
for (auto& th : threads) { for (auto& th : threads)
{
th.join(); th.join();
} }
@@ -130,7 +143,8 @@ TEST_F(CountingAllocatorTest, ThreadSafety) {
} }
// Test 6: Allocating zero objects should not change the counter // Test 6: Allocating zero objects should not change the counter
TEST_F(CountingAllocatorTest, AllocateZeroObjects) { TEST_F(CountingAllocatorTest, AllocateZeroObjects)
{
TestClass* ptr = allocator.allocate(0); TestClass* ptr = allocator.allocate(0);
EXPECT_NE(ptr, nullptr); EXPECT_NE(ptr, nullptr);
EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); EXPECT_EQ(allocatedMemory.load(), MemoryAllowance);

2
utils/common/countingallocator.h
View File

@@ -61,6 +61,7 @@ public:
T* ptr = static_cast<T*>(::operator new(n * sizeof(T))); T* ptr = static_cast<T*>(::operator new(n * sizeof(T)));
// std::cout << "[Allocate] " << n * sizeof(T) << " bytes at " << static_cast<void*>(ptr) // std::cout << "[Allocate] " << n * sizeof(T) << " bytes at " << static_cast<void*>(ptr)
// << ". current timit: " << std::dec << memoryLimitRef_.load() << std::hex << " bytes.\n"; // << ". current timit: " << std::dec << memoryLimitRef_.load() << std::hex << " bytes.\n";
// std::cout << std::dec;
return ptr; return ptr;
} }
@@ -87,6 +88,7 @@ public:
memoryLimitRef_.fetch_add(n * sizeof(T), std::memory_order_relaxed); memoryLimitRef_.fetch_add(n * sizeof(T), std::memory_order_relaxed);
// std::cout << "[Deallocate] " << n * sizeof(T) << " bytes from " << static_cast<void*>(ptr) // std::cout << "[Deallocate] " << n * sizeof(T) << " bytes from " << static_cast<void*>(ptr)
// << ". current timit: " << std::dec << memoryLimitRef_.load() << std::hex << " bytes.\n"; // << ". current timit: " << std::dec << memoryLimitRef_.load() << std::hex << " bytes.\n";
// std::cout << std::dec;
} }
// Equality operators (allocators are equal if they share the same counter) // Equality operators (allocators are equal if they share the same counter)

30
utils/rowgroup/rowgroup.cpp
View File

@@ -100,12 +100,12 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len)
if ((len + 4) >= CHUNK_SIZE) if ((len + 4) >= CHUNK_SIZE)
{ {
auto allocSize = len + sizeof(MemChunk) + 4; auto allocSize = len + sizeof(MemChunk) + 4;
if (alloc) // if (alloc)
{ // {
cout << "StringStore::storeString longStrings with alloc " << std::endl; // cout << "StringStore::storeString longStrings with alloc " << std::endl;
longStrings.emplace_back(std::allocate_shared<StringStoreBufType>(*alloc, allocSize)); // longStrings.emplace_back(std::allocate_shared<StringStoreBufType>(*alloc, allocSize));
} // }
else // else
{ {
cout << "StringStore::storeString longStrings no alloc " << std::endl; cout << "StringStore::storeString longStrings no alloc " << std::endl;
longStrings.emplace_back(std::make_shared<uint8_t[]>(allocSize)); longStrings.emplace_back(std::make_shared<uint8_t[]>(allocSize));
@@ -128,14 +128,14 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len)
if (alloc) if (alloc)
{ {
cout << "StringStore::storeString with alloc " << std::endl; cout << "StringStore::storeString with alloc " << std::endl;
mem.emplace_back(std::allocate_shared<StringStoreBufType>(*alloc, CHUNK_SIZE + sizeof(MemChunk))); mem.emplace_back(boost::allocate_shared<StringStoreBufType>(*alloc, CHUNK_SIZE + sizeof(MemChunk)));
// std::allocate_shared) newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); // boost::allocate_shared) newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]);
} }
else else
{ {
cout << "StringStore::storeString no alloc " << std::endl; cout << "StringStore::storeString no alloc " << std::endl;
mem.emplace_back(std::make_shared<uint8_t[]>(CHUNK_SIZE + sizeof(MemChunk))); mem.emplace_back(boost::make_shared<uint8_t[]>(CHUNK_SIZE + sizeof(MemChunk)));
// mem.emplace_back(std::allocate_shared<StringStoreBufType>(*alloc, CHUNK_SIZE + sizeof(MemChunk))); // mem.emplace_back(boost::allocate_shared<StringStoreBufType>(*alloc, CHUNK_SIZE + sizeof(MemChunk)));
// std::shared_ptr<uint8_t[]> newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); // std::shared_ptr<uint8_t[]> newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]);
} }
// mem.push_back(newOne); // mem.push_back(newOne);
@@ -209,12 +209,12 @@ void StringStore::deserialize(ByteStream& bs)
if (alloc) if (alloc)
{ {
cout << "StringStore::deserialize with alloc " << std::endl; cout << "StringStore::deserialize with alloc " << std::endl;
mem.emplace_back(std::allocate_shared<StringStoreBufType>(*alloc, size + sizeof(MemChunk))); mem.emplace_back(boost::allocate_shared<StringStoreBufType>(*alloc, size + sizeof(MemChunk)));
} }
else else
{ {
cout << "StringStore::deserialize no alloc " << std::endl; cout << "StringStore::deserialize no alloc " << std::endl;
mem.emplace_back(std::make_shared<uint8_t[]>(size + sizeof(MemChunk))); mem.emplace_back(boost::make_shared<uint8_t[]>(size + sizeof(MemChunk)));
} }
// mem[i].reset(new uint8_t[size + sizeof(MemChunk)]); // mem[i].reset(new uint8_t[size + sizeof(MemChunk)]);
mc = (MemChunk*)mem[i].get(); mc = (MemChunk*)mem[i].get();
@@ -230,7 +230,7 @@ void StringStore::deserialize(ByteStream& bs)
void StringStore::clear() void StringStore::clear()
{ {
vector<std::shared_ptr<uint8_t[]> > emptyv; vector<boost::shared_ptr<uint8_t[]> > emptyv;
vector<std::shared_ptr<uint8_t[]> > emptyv2; vector<std::shared_ptr<uint8_t[]> > emptyv2;
mem.swap(emptyv); mem.swap(emptyv);
longStrings.swap(emptyv2); longStrings.swap(emptyv2);
@@ -367,7 +367,7 @@ RGData::RGData(const RowGroup& rg, allocators::CountingAllocator<RGDataBufType>*
{ {
// rowData = shared_ptr<uint8_t[]>(buf, [alloc, allocSize](uint8_t* p) { alloc->deallocate(p, allocSize); // rowData = shared_ptr<uint8_t[]>(buf, [alloc, allocSize](uint8_t* p) { alloc->deallocate(p, allocSize);
// }); // });
rowData = std::allocate_shared<RGDataBufType>(*alloc, rg.getMaxDataSize()); rowData = boost::allocate_shared<RGDataBufType>(*alloc, rg.getMaxDataSize());
// rowData = std::make_shared(uint8_t[rg.getMaxDataSize()]); // rowData = std::make_shared(uint8_t[rg.getMaxDataSize()]);
if (rg.usesStringTable()) if (rg.usesStringTable())
@@ -381,7 +381,7 @@ void RGData::reinit(const RowGroup& rg, uint32_t rowCount)
if (alloc) if (alloc)
{ {
cout << "RGData::reinit with alloc " << std::endl; cout << "RGData::reinit with alloc " << std::endl;
rowData = std::allocate_shared<RGDataBufType>(*alloc, rg.getDataSize(rowCount)); rowData = boost::allocate_shared<RGDataBufType>(*alloc, rg.getDataSize(rowCount));
} }
else else
{ {

13
utils/rowgroup/rowgroup.h
View File

@@ -190,7 +190,7 @@ class StringStore
std::string empty_str; std::string empty_str;
static constexpr const uint32_t CHUNK_SIZE = 64 * 1024; // allocators like powers of 2 static constexpr const uint32_t CHUNK_SIZE = 64 * 1024; // allocators like powers of 2
std::vector<std::shared_ptr<uint8_t[]>> mem; std::vector<boost::shared_ptr<uint8_t[]>> mem;
// To store strings > 64KB (BLOB/TEXT) // To store strings > 64KB (BLOB/TEXT)
std::vector<std::shared_ptr<uint8_t[]>> longStrings; std::vector<std::shared_ptr<uint8_t[]>> longStrings;
@@ -286,7 +286,7 @@ class RGData
void clear(); void clear();
void reinit(const RowGroup& rg); void reinit(const RowGroup& rg);
void reinit(const RowGroup& rg, uint32_t rowCount); void reinit(const RowGroup& rg, uint32_t rowCount);
inline void setStringStore(std::shared_ptr<StringStore>& ss) inline void setStringStore(boost::shared_ptr<StringStore>& ss)
{ {
strings = ss; strings = ss;
} }
@@ -327,8 +327,8 @@ class RGData
private: private:
uint32_t rowSize = 0; // can't be. uint32_t rowSize = 0; // can't be.
uint32_t columnCount = 0; // shouldn't be, but... uint32_t columnCount = 0; // shouldn't be, but...
std::shared_ptr<RGDataBufType> rowData; boost::shared_ptr<RGDataBufType> rowData;
std::shared_ptr<StringStore> strings; boost::shared_ptr<StringStore> strings;
std::shared_ptr<UserDataStore> userDataStore; std::shared_ptr<UserDataStore> userDataStore;
allocators::CountingAllocator<RGDataBufType>* alloc = nullptr; allocators::CountingAllocator<RGDataBufType>* alloc = nullptr;
@@ -1599,7 +1599,7 @@ class RowGroup : public messageqcpp::Serializeable
const uint16_t& blockNum); const uint16_t& blockNum);
inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum); inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum);
inline void setStringStore(std::shared_ptr<StringStore>); inline void setStringStore(boost::shared_ptr<StringStore>);
const CHARSET_INFO* getCharset(uint32_t col); const CHARSET_INFO* getCharset(uint32_t col);
@@ -1911,7 +1911,8 @@ inline uint32_t RowGroup::getStringTableThreshold() const
return sTableThreshold; return sTableThreshold;
} }
inline void RowGroup::setStringStore(std::shared_ptr<StringStore> ss) // WIP mb unused
inline void RowGroup::setStringStore(boost::shared_ptr<StringStore> ss)
{ {
if (useStringTable) if (useStringTable)
{ {