1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

fix(perf,allocator): adding unit test to check CountingAllocator behavior when it is wrapped in STLPoolAllocator

This commit is contained in:
drrtuy
2025-04-25 22:03:51 +00:00
parent 671b7301f3
commit 252b3ddbef
8 changed files with 277 additions and 12 deletions

View File

@ -333,7 +333,12 @@ class ResourceManager
}
inline int64_t availableMemory() const
{
return totalUmMemLimit.load(std::memory_order_relaxed);
return atomicops::atomicLoadRef(totalUmMemLimit);
}
inline void setMemory(int64_t amount)
{
atomicops::atomicStoreRef(totalUmMemLimit, amount);
}
/* old HJ mem interface, used by HashJoin */

View File

@ -88,6 +88,15 @@ if (WITH_UNITTESTS)
target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
add_executable(stlpoolallocator stlpoolallocator.cpp)
target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
add_dependencies(stlpoolallocator googletest)
target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
add_executable(oid_server_return oid-server-return.cpp)
target_link_libraries(oid_server_return ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})
add_executable(comparators_tests comparators-tests.cpp)
target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
@ -110,4 +119,3 @@ if (WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark)
add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench)
endif()

View File

@ -202,7 +202,7 @@ TEST(PoolAllocatorTest, MultithreadedAllocationWithLock)
static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024;
// Test Fixture for AtomicCounterAllocator
// Test Fixture for CounterAllocator
class PoolallocatorTest : public ::testing::Test
{
protected:
@ -226,30 +226,30 @@ class PoolallocatorTest : public ::testing::Test
// Тест для проверки учёта потребления памяти в PoolAllocator.
TEST_F(PoolallocatorTest, AllocationWithAccounting)
{
int bufSize = 512;
int bufSize1 = 512;
const unsigned CUSTOM_SIZE = 1024;
PoolAllocator pa(allocator, CUSTOM_SIZE, false, true);
EXPECT_EQ(pa.getWindowSize(), CUSTOM_SIZE);
EXPECT_EQ(pa.getMemUsage(), 0ULL);
auto* ptr = pa.allocate(bufSize);
auto* ptr = pa.allocate(bufSize1);
EXPECT_NE(ptr, nullptr);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
pa.deallocate(ptr);
// B/c this PoolAllocator frees memory only when it's destroyed.
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
bufSize = 64536;
auto* ptr1 = pa.allocate(bufSize);
int bufSize2 = 64536;
auto* ptr1 = pa.allocate(bufSize2);
EXPECT_NE(ptr1, nullptr);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize2 - CUSTOM_SIZE);
pa.deallocate(ptr1);
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize);
EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize2);
}
TEST_F(PoolallocatorTest, MultithreadedAccountedAllocationWithLock)

226
tests/stlpoolallocator.cpp Normal file
View File

@ -0,0 +1,226 @@
#include <gtest/gtest.h>
#include <vector>
#include <thread>
#include <memory>
#include "stlpoolallocator.h"
#include "resourcemanager.h"
using namespace utils;
static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024;
/**
* Test fixture for STLPoolAllocator tests
*/
class STLPoolAllocatorTest : public ::testing::Test
{
protected:
using TestType = int; // Type we'll use for testing
using Allocator = STLPoolAllocator<TestType>;
};
/**
* Test default constructor and basic properties
*/
TEST_F(STLPoolAllocatorTest, DefaultConstructor)
{
Allocator alloc;
EXPECT_EQ(alloc.getMemUsage(), 0ULL);
EXPECT_EQ(alloc.max_size(), std::numeric_limits<uint64_t>::max() / sizeof(TestType));
}
/**
* Test basic allocation and deallocation
*/
TEST_F(STLPoolAllocatorTest, BasicAllocation)
{
Allocator alloc;
const size_t count = 10;
uint64_t initialUsage = alloc.getMemUsage();
// Allocate memory for 10 integers
TestType* ptr = alloc.allocate(count);
ASSERT_NE(ptr, nullptr);
EXPECT_GT(alloc.getMemUsage(), initialUsage);
// Construct objects
for (size_t i = 0; i < count; ++i)
{
alloc.construct(ptr + i, static_cast<TestType>(i));
}
// Verify constructed values
for (size_t i = 0; i < count; ++i)
{
EXPECT_EQ(ptr[i], static_cast<TestType>(i));
}
// Destroy objects
for (size_t i = 0; i < count; ++i)
{
alloc.destroy(ptr + i);
}
// Deallocate memory
alloc.deallocate(ptr, count);
}
/**
* Test copy constructor and assignment operator
*/
TEST_F(STLPoolAllocatorTest, CopyAndAssignment)
{
Allocator alloc1;
TestType* ptr = alloc1.allocate(1);
alloc1.construct(ptr, 42);
// Test copy constructor
Allocator alloc2(alloc1);
EXPECT_EQ(alloc1.getMemUsage(), alloc2.getMemUsage());
// Test assignment operator
Allocator alloc3;
alloc3 = alloc1;
EXPECT_EQ(alloc1.getMemUsage(), alloc3.getMemUsage());
alloc1.destroy(ptr);
alloc1.deallocate(ptr, 1);
}
/**
* Test STL container integration (vector)
*/
TEST_F(STLPoolAllocatorTest, VectorIntegration)
{
std::vector<TestType, Allocator> vec;
const size_t testSize = 1000;
// Reserve to avoid reallocation
vec.reserve(testSize);
// Add elements
for (size_t i = 0; i < testSize; ++i)
{
vec.push_back(static_cast<TestType>(i));
}
// Verify elements
for (size_t i = 0; i < testSize; ++i)
{
EXPECT_EQ(vec[i], static_cast<TestType>(i));
}
// Clear vector
vec.clear();
vec.shrink_to_fit();
}
// /**
// * Test multithreaded allocation
// */
// TEST_F(STLPoolAllocatorTest, MultithreadedAllocation)
// {
// const int THREAD_COUNT = 4;
// const size_t ALLOC_SIZE = 100;
// std::vector<std::thread> threads;
// Allocator alloc;
// uint64_t initialUsage = alloc.getMemUsage();
// // Create threads that will allocate memory
// for (int i = 0; i < THREAD_COUNT; ++i)
// {
// threads.emplace_back(
// [&alloc]()
// {
// std::vector<TestType*, STLPoolAllocator<TestType*>> ptrs(alloc);
// for (size_t j = 0; j < 10; ++j)
// {
// TestType* ptr = alloc.allocate(ALLOC_SIZE);
// for (size_t k = 0; k < ALLOC_SIZE; ++k)
// {
// alloc.construct(ptr + k, static_cast<TestType>(k));
// }
// ptrs.push_back(ptr);
// }
// // Cleanup
// for (auto ptr : ptrs)
// {
// for (size_t k = 0; k < ALLOC_SIZE; ++k)
// {
// alloc.destroy(ptr + k);
// }
// alloc.deallocate(ptr, ALLOC_SIZE);
// }
// });
// }
// // Wait for all threads to complete
// for (auto& th : threads)
// {
// th.join();
// }
// // Memory usage should be greater than initial due to vector allocations
// EXPECT_GT(alloc.getMemUsage(), initialUsage);
// }
/**
* Test ResourceManager integration
*/
TEST_F(STLPoolAllocatorTest, ResourceManagerIntegration)
{
joblist::ResourceManager rm(true, nullptr);
// To set the memory allowance
rm.setMemory(MemoryAllowance);
std::cout << "Memory allowance: " << MemoryAllowance << std::endl;
Allocator alloc(&rm);
std::cout << "Memory available 1 : " << rm.availableMemory() << std::endl;
// Basic allocation test with ResourceManager
TestType* ptr = alloc.allocate(1);
ASSERT_NE(ptr, nullptr);
std::cout << "Memory available 2 : " << rm.availableMemory() << std::endl;
alloc.construct(ptr, 42);
EXPECT_EQ(*ptr, 42);
alloc.destroy(ptr);
alloc.deallocate(ptr, 1);
std::cout << "Memory available 3 : " << rm.availableMemory() << std::endl;
TestType* ptr2 = alloc.allocate(65537);
ASSERT_NE(ptr2, nullptr);
alloc.construct(ptr2, 42);
EXPECT_EQ(*ptr2, 42);
std::cout << "Memory available 4 : " << rm.availableMemory() << std::endl;
alloc.destroy(ptr2);
alloc.deallocate(ptr2, 1);
std::cout << "Memory available 5 : " << rm.availableMemory() << std::endl;
}
/**
* Test rebind capability (required by STL)
*/
TEST_F(STLPoolAllocatorTest, RebindTest)
{
using DoubleAllocator = typename Allocator::template rebind<double>::other;
DoubleAllocator doubleAlloc;
// Allocate and construct a double
double* ptr = doubleAlloc.allocate(1);
ASSERT_NE(ptr, nullptr);
doubleAlloc.construct(ptr, 3.14159);
EXPECT_DOUBLE_EQ(*ptr, 3.14159);
doubleAlloc.destroy(ptr);
doubleAlloc.deallocate(ptr, 1);
}

View File

@ -41,6 +41,16 @@ inline int64_t atomicSubRef(std::atomic<int64_t>& ref, int64_t val)
{
return ref.fetch_sub(val, std::memory_order_relaxed);
}
inline void atomicStoreRef(std::atomic<int64_t>& ref, int64_t val)
{
ref.store(val, std::memory_order_relaxed);
}
inline int64_t atomicLoadRef(const std::atomic<int64_t>& ref)
{
return ref.load(std::memory_order_relaxed);
}
// Returns the resulting, incremented value
template <typename T>
inline T atomicInc(volatile T* mem)

View File

@ -25,6 +25,8 @@
#include <boost/smart_ptr/allocate_shared_array.hpp>
#include <boost/smart_ptr/make_shared_array.hpp>
#include <iostream>
#include "poolallocator.h"
using namespace std;
@ -34,6 +36,7 @@ namespace utils
{
PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v)
{
std::cout << "PoolAllocator copy assignment" << std::endl;
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
useLock = v.useLock;
@ -54,15 +57,19 @@ void PoolAllocator::deallocateAll()
void PoolAllocator::newBlock()
{
capacityRemaining = allocSize;
std::cout << "PoolAllocator new block" << std::endl;
if (!tmpSpace || mem.size() == 0)
{
if (alloc)
{
std::cout << "PoolAllocator new block with counting alloc" << std::endl;
mem.emplace_back(boost::allocate_shared<PoolAllocatorBufType>(*alloc, allocSize));
}
else
{
std::cout << "PoolAllocator new block w/o counting alloc" << std::endl;
mem.emplace_back(boost::make_shared<PoolAllocatorBufType>(allocSize));
}
nextAlloc = mem.back().get();
@ -74,14 +81,17 @@ void PoolAllocator::newBlock()
void* PoolAllocator::allocOOB(uint64_t size)
{
OOBMemInfo memInfo;
std::cout << "PoolAllocator allocOOB" << std::endl;
memUsage += size;
if (alloc)
{
std::cout << "PoolAllocator allocOOB with counting alloc" << std::endl;
memInfo.mem = boost::allocate_shared<PoolAllocatorBufType>(*alloc, size);
}
else
{
std::cout << "PoolAllocator allocOOB w/o counting alloc" << std::endl;
memInfo.mem = boost::make_shared<PoolAllocatorBufType>(size);
}
memInfo.size = size;

View File

@ -25,6 +25,7 @@
#pragma once
#include <atomic>
#include <unistd.h>
#include <stdint.h>
#include <optional>
@ -32,9 +33,10 @@
#include <map>
#include <memory>
#include <iostream>
#include <boost/smart_ptr/allocate_shared_array.hpp>
#include <atomic>
#include "countingallocator.h"
@ -57,6 +59,7 @@ class PoolAllocator
, useLock(_useLock)
, lock(false)
{
std::cout << "PoolAllocator w/o counting allocator created" << std::endl;
}
PoolAllocator(allocators::CountingAllocator<PoolAllocatorBufType> alloc, unsigned windowSize = DEFAULT_WINDOW_SIZE,
bool isTmpSpace = false, bool _useLock = false)
@ -69,6 +72,7 @@ class PoolAllocator
, lock(false)
, alloc(alloc)
{
std::cout << "PoolAllocator with counting allocator created" << std::endl;
}
PoolAllocator(const PoolAllocator& p)
: allocSize(p.allocSize)

View File

@ -101,11 +101,13 @@ STLPoolAllocator<T>::STLPoolAllocator(joblist::ResourceManager* rm)
{
if (rm)
{
std::cout << "STLPoolAllocator with RM " << std::endl;
auto alloc = rm->getAllocator<PoolAllocatorBufType>();
pa.reset(new PoolAllocator(alloc, DEFAULT_SIZE));
}
else
{
std::cout << "STLPoolAllocator w/o RM " << std::endl;
pa.reset(new PoolAllocator(DEFAULT_SIZE));
}
}