You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
fix(perf,allocator): adding unit test to check CountingAllocator behavior when it is wrapped in STLPoolAllocator
This commit is contained in:
@ -333,7 +333,12 @@ class ResourceManager
|
||||
}
|
||||
inline int64_t availableMemory() const
|
||||
{
|
||||
return totalUmMemLimit.load(std::memory_order_relaxed);
|
||||
return atomicops::atomicLoadRef(totalUmMemLimit);
|
||||
}
|
||||
|
||||
inline void setMemory(int64_t amount)
|
||||
{
|
||||
atomicops::atomicStoreRef(totalUmMemLimit, amount);
|
||||
}
|
||||
|
||||
/* old HJ mem interface, used by HashJoin */
|
||||
|
@ -88,6 +88,15 @@ if (WITH_UNITTESTS)
|
||||
target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
|
||||
gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
|
||||
|
||||
add_executable(stlpoolallocator stlpoolallocator.cpp)
|
||||
target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
|
||||
add_dependencies(stlpoolallocator googletest)
|
||||
target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
|
||||
gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
|
||||
|
||||
add_executable(oid_server_return oid-server-return.cpp)
|
||||
target_link_libraries(oid_server_return ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})
|
||||
|
||||
add_executable(comparators_tests comparators-tests.cpp)
|
||||
target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
|
||||
add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
|
||||
@ -110,4 +119,3 @@ if (WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
|
||||
target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark)
|
||||
add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench)
|
||||
endif()
|
||||
|
||||
|
@ -202,7 +202,7 @@ TEST(PoolAllocatorTest, MultithreadedAllocationWithLock)
|
||||
|
||||
static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024;
|
||||
|
||||
// Test Fixture for AtomicCounterAllocator
|
||||
// Test Fixture for CounterAllocator
|
||||
class PoolallocatorTest : public ::testing::Test
|
||||
{
|
||||
protected:
|
||||
@ -226,30 +226,30 @@ class PoolallocatorTest : public ::testing::Test
|
||||
// Тест для проверки учёта потребления памяти в PoolAllocator.
|
||||
TEST_F(PoolallocatorTest, AllocationWithAccounting)
|
||||
{
|
||||
int bufSize = 512;
|
||||
int bufSize1 = 512;
|
||||
const unsigned CUSTOM_SIZE = 1024;
|
||||
PoolAllocator pa(allocator, CUSTOM_SIZE, false, true);
|
||||
EXPECT_EQ(pa.getWindowSize(), CUSTOM_SIZE);
|
||||
EXPECT_EQ(pa.getMemUsage(), 0ULL);
|
||||
auto* ptr = pa.allocate(bufSize);
|
||||
auto* ptr = pa.allocate(bufSize1);
|
||||
|
||||
EXPECT_NE(ptr, nullptr);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
|
||||
pa.deallocate(ptr);
|
||||
// B/c this PoolAllocator frees memory only when it's destroyed.
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
|
||||
|
||||
bufSize = 64536;
|
||||
auto* ptr1 = pa.allocate(bufSize);
|
||||
int bufSize2 = 64536;
|
||||
auto* ptr1 = pa.allocate(bufSize2);
|
||||
|
||||
EXPECT_NE(ptr1, nullptr);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize2 - CUSTOM_SIZE);
|
||||
|
||||
pa.deallocate(ptr1);
|
||||
EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
|
||||
EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize);
|
||||
EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize2);
|
||||
}
|
||||
|
||||
TEST_F(PoolallocatorTest, MultithreadedAccountedAllocationWithLock)
|
||||
|
226
tests/stlpoolallocator.cpp
Normal file
226
tests/stlpoolallocator.cpp
Normal file
@ -0,0 +1,226 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
#include "stlpoolallocator.h"
|
||||
#include "resourcemanager.h"
|
||||
|
||||
using namespace utils;
|
||||
|
||||
static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024;
|
||||
|
||||
|
||||
/**
|
||||
* Test fixture for STLPoolAllocator tests
|
||||
*/
|
||||
class STLPoolAllocatorTest : public ::testing::Test
|
||||
{
|
||||
protected:
|
||||
using TestType = int; // Type we'll use for testing
|
||||
using Allocator = STLPoolAllocator<TestType>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Test default constructor and basic properties
|
||||
*/
|
||||
TEST_F(STLPoolAllocatorTest, DefaultConstructor)
|
||||
{
|
||||
Allocator alloc;
|
||||
EXPECT_EQ(alloc.getMemUsage(), 0ULL);
|
||||
EXPECT_EQ(alloc.max_size(), std::numeric_limits<uint64_t>::max() / sizeof(TestType));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test basic allocation and deallocation
|
||||
*/
|
||||
TEST_F(STLPoolAllocatorTest, BasicAllocation)
|
||||
{
|
||||
Allocator alloc;
|
||||
const size_t count = 10;
|
||||
uint64_t initialUsage = alloc.getMemUsage();
|
||||
|
||||
// Allocate memory for 10 integers
|
||||
TestType* ptr = alloc.allocate(count);
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
EXPECT_GT(alloc.getMemUsage(), initialUsage);
|
||||
|
||||
// Construct objects
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
alloc.construct(ptr + i, static_cast<TestType>(i));
|
||||
}
|
||||
|
||||
// Verify constructed values
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
EXPECT_EQ(ptr[i], static_cast<TestType>(i));
|
||||
}
|
||||
|
||||
// Destroy objects
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
alloc.destroy(ptr + i);
|
||||
}
|
||||
|
||||
// Deallocate memory
|
||||
alloc.deallocate(ptr, count);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test copy constructor and assignment operator
|
||||
*/
|
||||
TEST_F(STLPoolAllocatorTest, CopyAndAssignment)
|
||||
{
|
||||
Allocator alloc1;
|
||||
TestType* ptr = alloc1.allocate(1);
|
||||
alloc1.construct(ptr, 42);
|
||||
|
||||
// Test copy constructor
|
||||
Allocator alloc2(alloc1);
|
||||
EXPECT_EQ(alloc1.getMemUsage(), alloc2.getMemUsage());
|
||||
|
||||
// Test assignment operator
|
||||
Allocator alloc3;
|
||||
alloc3 = alloc1;
|
||||
EXPECT_EQ(alloc1.getMemUsage(), alloc3.getMemUsage());
|
||||
|
||||
alloc1.destroy(ptr);
|
||||
alloc1.deallocate(ptr, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test STL container integration (vector)
|
||||
*/
|
||||
TEST_F(STLPoolAllocatorTest, VectorIntegration)
|
||||
{
|
||||
std::vector<TestType, Allocator> vec;
|
||||
const size_t testSize = 1000;
|
||||
|
||||
// Reserve to avoid reallocation
|
||||
vec.reserve(testSize);
|
||||
|
||||
// Add elements
|
||||
for (size_t i = 0; i < testSize; ++i)
|
||||
{
|
||||
vec.push_back(static_cast<TestType>(i));
|
||||
}
|
||||
|
||||
// Verify elements
|
||||
for (size_t i = 0; i < testSize; ++i)
|
||||
{
|
||||
EXPECT_EQ(vec[i], static_cast<TestType>(i));
|
||||
}
|
||||
|
||||
// Clear vector
|
||||
vec.clear();
|
||||
vec.shrink_to_fit();
|
||||
}
|
||||
|
||||
// /**
|
||||
// * Test multithreaded allocation
|
||||
// */
|
||||
// TEST_F(STLPoolAllocatorTest, MultithreadedAllocation)
|
||||
// {
|
||||
// const int THREAD_COUNT = 4;
|
||||
// const size_t ALLOC_SIZE = 100;
|
||||
// std::vector<std::thread> threads;
|
||||
|
||||
// Allocator alloc;
|
||||
// uint64_t initialUsage = alloc.getMemUsage();
|
||||
|
||||
// // Create threads that will allocate memory
|
||||
// for (int i = 0; i < THREAD_COUNT; ++i)
|
||||
// {
|
||||
// threads.emplace_back(
|
||||
// [&alloc]()
|
||||
// {
|
||||
// std::vector<TestType*, STLPoolAllocator<TestType*>> ptrs(alloc);
|
||||
|
||||
// for (size_t j = 0; j < 10; ++j)
|
||||
// {
|
||||
// TestType* ptr = alloc.allocate(ALLOC_SIZE);
|
||||
// for (size_t k = 0; k < ALLOC_SIZE; ++k)
|
||||
// {
|
||||
// alloc.construct(ptr + k, static_cast<TestType>(k));
|
||||
// }
|
||||
// ptrs.push_back(ptr);
|
||||
// }
|
||||
|
||||
// // Cleanup
|
||||
// for (auto ptr : ptrs)
|
||||
// {
|
||||
// for (size_t k = 0; k < ALLOC_SIZE; ++k)
|
||||
// {
|
||||
// alloc.destroy(ptr + k);
|
||||
// }
|
||||
// alloc.deallocate(ptr, ALLOC_SIZE);
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
// // Wait for all threads to complete
|
||||
// for (auto& th : threads)
|
||||
// {
|
||||
// th.join();
|
||||
// }
|
||||
|
||||
// // Memory usage should be greater than initial due to vector allocations
|
||||
// EXPECT_GT(alloc.getMemUsage(), initialUsage);
|
||||
// }
|
||||
|
||||
/**
|
||||
* Test ResourceManager integration
|
||||
*/
|
||||
TEST_F(STLPoolAllocatorTest, ResourceManagerIntegration)
|
||||
{
|
||||
joblist::ResourceManager rm(true, nullptr);
|
||||
// To set the memory allowance
|
||||
rm.setMemory(MemoryAllowance);
|
||||
std::cout << "Memory allowance: " << MemoryAllowance << std::endl;
|
||||
|
||||
Allocator alloc(&rm);
|
||||
|
||||
std::cout << "Memory available 1 : " << rm.availableMemory() << std::endl;
|
||||
// Basic allocation test with ResourceManager
|
||||
TestType* ptr = alloc.allocate(1);
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
std::cout << "Memory available 2 : " << rm.availableMemory() << std::endl;
|
||||
|
||||
|
||||
alloc.construct(ptr, 42);
|
||||
EXPECT_EQ(*ptr, 42);
|
||||
|
||||
alloc.destroy(ptr);
|
||||
alloc.deallocate(ptr, 1);
|
||||
std::cout << "Memory available 3 : " << rm.availableMemory() << std::endl;
|
||||
|
||||
TestType* ptr2 = alloc.allocate(65537);
|
||||
ASSERT_NE(ptr2, nullptr);
|
||||
alloc.construct(ptr2, 42);
|
||||
EXPECT_EQ(*ptr2, 42);
|
||||
std::cout << "Memory available 4 : " << rm.availableMemory() << std::endl;
|
||||
|
||||
|
||||
alloc.destroy(ptr2);
|
||||
alloc.deallocate(ptr2, 1);
|
||||
std::cout << "Memory available 5 : " << rm.availableMemory() << std::endl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test rebind capability (required by STL)
|
||||
*/
|
||||
TEST_F(STLPoolAllocatorTest, RebindTest)
|
||||
{
|
||||
using DoubleAllocator = typename Allocator::template rebind<double>::other;
|
||||
DoubleAllocator doubleAlloc;
|
||||
|
||||
// Allocate and construct a double
|
||||
double* ptr = doubleAlloc.allocate(1);
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
|
||||
doubleAlloc.construct(ptr, 3.14159);
|
||||
EXPECT_DOUBLE_EQ(*ptr, 3.14159);
|
||||
|
||||
doubleAlloc.destroy(ptr);
|
||||
doubleAlloc.deallocate(ptr, 1);
|
||||
}
|
@ -41,6 +41,16 @@ inline int64_t atomicSubRef(std::atomic<int64_t>& ref, int64_t val)
|
||||
{
|
||||
return ref.fetch_sub(val, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
inline void atomicStoreRef(std::atomic<int64_t>& ref, int64_t val)
|
||||
{
|
||||
ref.store(val, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
inline int64_t atomicLoadRef(const std::atomic<int64_t>& ref)
|
||||
{
|
||||
return ref.load(std::memory_order_relaxed);
|
||||
}
|
||||
// Returns the resulting, incremented value
|
||||
template <typename T>
|
||||
inline T atomicInc(volatile T* mem)
|
||||
|
@ -25,6 +25,8 @@
|
||||
#include <boost/smart_ptr/allocate_shared_array.hpp>
|
||||
#include <boost/smart_ptr/make_shared_array.hpp>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "poolallocator.h"
|
||||
|
||||
using namespace std;
|
||||
@ -34,6 +36,7 @@ namespace utils
|
||||
{
|
||||
PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v)
|
||||
{
|
||||
std::cout << "PoolAllocator copy assignment" << std::endl;
|
||||
allocSize = v.allocSize;
|
||||
tmpSpace = v.tmpSpace;
|
||||
useLock = v.useLock;
|
||||
@ -54,15 +57,19 @@ void PoolAllocator::deallocateAll()
|
||||
void PoolAllocator::newBlock()
|
||||
{
|
||||
capacityRemaining = allocSize;
|
||||
std::cout << "PoolAllocator new block" << std::endl;
|
||||
|
||||
if (!tmpSpace || mem.size() == 0)
|
||||
{
|
||||
if (alloc)
|
||||
{
|
||||
std::cout << "PoolAllocator new block with counting alloc" << std::endl;
|
||||
|
||||
mem.emplace_back(boost::allocate_shared<PoolAllocatorBufType>(*alloc, allocSize));
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "PoolAllocator new block w/o counting alloc" << std::endl;
|
||||
mem.emplace_back(boost::make_shared<PoolAllocatorBufType>(allocSize));
|
||||
}
|
||||
nextAlloc = mem.back().get();
|
||||
@ -74,14 +81,17 @@ void PoolAllocator::newBlock()
|
||||
void* PoolAllocator::allocOOB(uint64_t size)
|
||||
{
|
||||
OOBMemInfo memInfo;
|
||||
std::cout << "PoolAllocator allocOOB" << std::endl;
|
||||
|
||||
memUsage += size;
|
||||
if (alloc)
|
||||
{
|
||||
std::cout << "PoolAllocator allocOOB with counting alloc" << std::endl;
|
||||
memInfo.mem = boost::allocate_shared<PoolAllocatorBufType>(*alloc, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "PoolAllocator allocOOB w/o counting alloc" << std::endl;
|
||||
memInfo.mem = boost::make_shared<PoolAllocatorBufType>(size);
|
||||
}
|
||||
memInfo.size = size;
|
||||
|
@ -25,6 +25,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <unistd.h>
|
||||
#include <stdint.h>
|
||||
#include <optional>
|
||||
@ -32,9 +33,10 @@
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <boost/smart_ptr/allocate_shared_array.hpp>
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "countingallocator.h"
|
||||
|
||||
@ -57,6 +59,7 @@ class PoolAllocator
|
||||
, useLock(_useLock)
|
||||
, lock(false)
|
||||
{
|
||||
std::cout << "PoolAllocator w/o counting allocator created" << std::endl;
|
||||
}
|
||||
PoolAllocator(allocators::CountingAllocator<PoolAllocatorBufType> alloc, unsigned windowSize = DEFAULT_WINDOW_SIZE,
|
||||
bool isTmpSpace = false, bool _useLock = false)
|
||||
@ -69,6 +72,7 @@ class PoolAllocator
|
||||
, lock(false)
|
||||
, alloc(alloc)
|
||||
{
|
||||
std::cout << "PoolAllocator with counting allocator created" << std::endl;
|
||||
}
|
||||
PoolAllocator(const PoolAllocator& p)
|
||||
: allocSize(p.allocSize)
|
||||
|
@ -101,11 +101,13 @@ STLPoolAllocator<T>::STLPoolAllocator(joblist::ResourceManager* rm)
|
||||
{
|
||||
if (rm)
|
||||
{
|
||||
std::cout << "STLPoolAllocator with RM " << std::endl;
|
||||
auto alloc = rm->getAllocator<PoolAllocatorBufType>();
|
||||
pa.reset(new PoolAllocator(alloc, DEFAULT_SIZE));
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "STLPoolAllocator w/o RM " << std::endl;
|
||||
pa.reset(new PoolAllocator(DEFAULT_SIZE));
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user