diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 396cbe2c5..b5d1cea57 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -333,9 +333,14 @@ class ResourceManager } inline int64_t availableMemory() const { - return totalUmMemLimit.load(std::memory_order_relaxed); + return atomicops::atomicLoadRef(totalUmMemLimit); } + inline void setMemory(int64_t amount) + { + atomicops::atomicStoreRef(totalUmMemLimit, amount); + } + /* old HJ mem interface, used by HashJoin */ uint64_t getHjPmMaxMemorySmallSide(uint32_t sessionID) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1110e13e0..f4d95fd38 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -88,6 +88,15 @@ if (WITH_UNITTESTS) target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:) + add_executable(stlpoolallocator stlpoolallocator.cpp) + target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare) + add_dependencies(stlpoolallocator googletest) + target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) + gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:) + + add_executable(oid_server_return oid-server-return.cpp) + target_link_libraries(oid_server_return ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS}) + add_executable(comparators_tests comparators-tests.cpp) target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:comparators_tests COMMAND comparators_tests) @@ -110,4 +119,3 @@ if (WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")) target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark) add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench) endif() - diff --git a/tests/poolallocator.cpp b/tests/poolallocator.cpp index e0ec5fae4..b4272f544 100644 --- a/tests/poolallocator.cpp +++ b/tests/poolallocator.cpp @@ -202,7 +202,7 @@ TEST(PoolAllocatorTest, MultithreadedAllocationWithLock) static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024; -// Test Fixture for AtomicCounterAllocator +// Test Fixture for CounterAllocator class PoolallocatorTest : public ::testing::Test { protected: @@ -226,30 +226,30 @@ class PoolallocatorTest : public ::testing::Test // Тест для проверки учёта потребления памяти в PoolAllocator. TEST_F(PoolallocatorTest, AllocationWithAccounting) { - int bufSize = 512; + int bufSize1 = 512; const unsigned CUSTOM_SIZE = 1024; PoolAllocator pa(allocator, CUSTOM_SIZE, false, true); EXPECT_EQ(pa.getWindowSize(), CUSTOM_SIZE); EXPECT_EQ(pa.getMemUsage(), 0ULL); - auto* ptr = pa.allocate(bufSize); + auto* ptr = pa.allocate(bufSize1); EXPECT_NE(ptr, nullptr); - EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize); + EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1); EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE); pa.deallocate(ptr); // B/c this PoolAllocator frees memory only when it's destroyed. - EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize); + EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1); EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE); - bufSize = 64536; - auto* ptr1 = pa.allocate(bufSize); + int bufSize2 = 64536; + auto* ptr1 = pa.allocate(bufSize2); EXPECT_NE(ptr1, nullptr); - EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize); + EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize2 - CUSTOM_SIZE); pa.deallocate(ptr1); EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE); - EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize); + EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize2); } TEST_F(PoolallocatorTest, MultithreadedAccountedAllocationWithLock) diff --git a/tests/stlpoolallocator.cpp b/tests/stlpoolallocator.cpp new file mode 100644 index 000000000..47159d68e --- /dev/null +++ b/tests/stlpoolallocator.cpp @@ -0,0 +1,226 @@ +#include +#include +#include +#include +#include "stlpoolallocator.h" +#include "resourcemanager.h" + +using namespace utils; + +static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024; + + +/** + * Test fixture for STLPoolAllocator tests + */ +class STLPoolAllocatorTest : public ::testing::Test +{ +protected: + using TestType = int; // Type we'll use for testing + using Allocator = STLPoolAllocator; +}; + +/** + * Test default constructor and basic properties + */ +TEST_F(STLPoolAllocatorTest, DefaultConstructor) +{ + Allocator alloc; + EXPECT_EQ(alloc.getMemUsage(), 0ULL); + EXPECT_EQ(alloc.max_size(), std::numeric_limits::max() / sizeof(TestType)); +} + +/** + * Test basic allocation and deallocation + */ +TEST_F(STLPoolAllocatorTest, BasicAllocation) +{ + Allocator alloc; + const size_t count = 10; + uint64_t initialUsage = alloc.getMemUsage(); + + // Allocate memory for 10 integers + TestType* ptr = alloc.allocate(count); + ASSERT_NE(ptr, nullptr); + EXPECT_GT(alloc.getMemUsage(), initialUsage); + + // Construct objects + for (size_t i = 0; i < count; ++i) + { + alloc.construct(ptr + i, static_cast(i)); + } + + // Verify constructed values + for (size_t i = 0; i < count; ++i) + { + EXPECT_EQ(ptr[i], static_cast(i)); + } + + // Destroy objects + for (size_t i = 0; i < count; ++i) + { + alloc.destroy(ptr + i); + } + + // Deallocate memory + alloc.deallocate(ptr, count); +} + +/** + * Test copy constructor and assignment operator + */ +TEST_F(STLPoolAllocatorTest, CopyAndAssignment) +{ + Allocator alloc1; + TestType* ptr = alloc1.allocate(1); + alloc1.construct(ptr, 42); + + // Test copy constructor + Allocator alloc2(alloc1); + EXPECT_EQ(alloc1.getMemUsage(), alloc2.getMemUsage()); + + // Test assignment operator + Allocator alloc3; + alloc3 = alloc1; + EXPECT_EQ(alloc1.getMemUsage(), alloc3.getMemUsage()); + + alloc1.destroy(ptr); + alloc1.deallocate(ptr, 1); +} + +/** + * Test STL container integration (vector) + */ +TEST_F(STLPoolAllocatorTest, VectorIntegration) +{ + std::vector vec; + const size_t testSize = 1000; + + // Reserve to avoid reallocation + vec.reserve(testSize); + + // Add elements + for (size_t i = 0; i < testSize; ++i) + { + vec.push_back(static_cast(i)); + } + + // Verify elements + for (size_t i = 0; i < testSize; ++i) + { + EXPECT_EQ(vec[i], static_cast(i)); + } + + // Clear vector + vec.clear(); + vec.shrink_to_fit(); +} + +// /** +// * Test multithreaded allocation +// */ +// TEST_F(STLPoolAllocatorTest, MultithreadedAllocation) +// { +// const int THREAD_COUNT = 4; +// const size_t ALLOC_SIZE = 100; +// std::vector threads; + +// Allocator alloc; +// uint64_t initialUsage = alloc.getMemUsage(); + +// // Create threads that will allocate memory +// for (int i = 0; i < THREAD_COUNT; ++i) +// { +// threads.emplace_back( +// [&alloc]() +// { +// std::vector> ptrs(alloc); + +// for (size_t j = 0; j < 10; ++j) +// { +// TestType* ptr = alloc.allocate(ALLOC_SIZE); +// for (size_t k = 0; k < ALLOC_SIZE; ++k) +// { +// alloc.construct(ptr + k, static_cast(k)); +// } +// ptrs.push_back(ptr); +// } + +// // Cleanup +// for (auto ptr : ptrs) +// { +// for (size_t k = 0; k < ALLOC_SIZE; ++k) +// { +// alloc.destroy(ptr + k); +// } +// alloc.deallocate(ptr, ALLOC_SIZE); +// } +// }); +// } + +// // Wait for all threads to complete +// for (auto& th : threads) +// { +// th.join(); +// } + +// // Memory usage should be greater than initial due to vector allocations +// EXPECT_GT(alloc.getMemUsage(), initialUsage); +// } + +/** + * Test ResourceManager integration + */ +TEST_F(STLPoolAllocatorTest, ResourceManagerIntegration) +{ + joblist::ResourceManager rm(true, nullptr); + // To set the memory allowance + rm.setMemory(MemoryAllowance); + std::cout << "Memory allowance: " << MemoryAllowance << std::endl; + + Allocator alloc(&rm); + + std::cout << "Memory available 1 : " << rm.availableMemory() << std::endl; + // Basic allocation test with ResourceManager + TestType* ptr = alloc.allocate(1); + ASSERT_NE(ptr, nullptr); + std::cout << "Memory available 2 : " << rm.availableMemory() << std::endl; + + + alloc.construct(ptr, 42); + EXPECT_EQ(*ptr, 42); + + alloc.destroy(ptr); + alloc.deallocate(ptr, 1); + std::cout << "Memory available 3 : " << rm.availableMemory() << std::endl; + + TestType* ptr2 = alloc.allocate(65537); + ASSERT_NE(ptr2, nullptr); + alloc.construct(ptr2, 42); + EXPECT_EQ(*ptr2, 42); + std::cout << "Memory available 4 : " << rm.availableMemory() << std::endl; + + + alloc.destroy(ptr2); + alloc.deallocate(ptr2, 1); + std::cout << "Memory available 5 : " << rm.availableMemory() << std::endl; +} + +/** + * Test rebind capability (required by STL) + */ +TEST_F(STLPoolAllocatorTest, RebindTest) +{ + using DoubleAllocator = typename Allocator::template rebind::other; + DoubleAllocator doubleAlloc; + + // Allocate and construct a double + double* ptr = doubleAlloc.allocate(1); + ASSERT_NE(ptr, nullptr); + + doubleAlloc.construct(ptr, 3.14159); + EXPECT_DOUBLE_EQ(*ptr, 3.14159); + + doubleAlloc.destroy(ptr); + doubleAlloc.deallocate(ptr, 1); +} diff --git a/utils/common/atomicops.h b/utils/common/atomicops.h index 9aec08da9..1f8b60d1a 100644 --- a/utils/common/atomicops.h +++ b/utils/common/atomicops.h @@ -41,6 +41,16 @@ inline int64_t atomicSubRef(std::atomic& ref, int64_t val) { return ref.fetch_sub(val, std::memory_order_relaxed); } + +inline void atomicStoreRef(std::atomic& ref, int64_t val) +{ + ref.store(val, std::memory_order_relaxed); +} + +inline int64_t atomicLoadRef(const std::atomic& ref) +{ + return ref.load(std::memory_order_relaxed); +} // Returns the resulting, incremented value template inline T atomicInc(volatile T* mem) diff --git a/utils/common/poolallocator.cpp b/utils/common/poolallocator.cpp index aa6449cc7..428943d13 100644 --- a/utils/common/poolallocator.cpp +++ b/utils/common/poolallocator.cpp @@ -25,6 +25,8 @@ #include #include +#include + #include "poolallocator.h" using namespace std; @@ -34,6 +36,7 @@ namespace utils { PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v) { + std::cout << "PoolAllocator copy assignment" << std::endl; allocSize = v.allocSize; tmpSpace = v.tmpSpace; useLock = v.useLock; @@ -54,15 +57,19 @@ void PoolAllocator::deallocateAll() void PoolAllocator::newBlock() { capacityRemaining = allocSize; + std::cout << "PoolAllocator new block" << std::endl; if (!tmpSpace || mem.size() == 0) { if (alloc) { + std::cout << "PoolAllocator new block with counting alloc" << std::endl; + mem.emplace_back(boost::allocate_shared(*alloc, allocSize)); } else { + std::cout << "PoolAllocator new block w/o counting alloc" << std::endl; mem.emplace_back(boost::make_shared(allocSize)); } nextAlloc = mem.back().get(); @@ -74,14 +81,17 @@ void PoolAllocator::newBlock() void* PoolAllocator::allocOOB(uint64_t size) { OOBMemInfo memInfo; + std::cout << "PoolAllocator allocOOB" << std::endl; memUsage += size; if (alloc) { + std::cout << "PoolAllocator allocOOB with counting alloc" << std::endl; memInfo.mem = boost::allocate_shared(*alloc, size); } else { + std::cout << "PoolAllocator allocOOB w/o counting alloc" << std::endl; memInfo.mem = boost::make_shared(size); } memInfo.size = size; diff --git a/utils/common/poolallocator.h b/utils/common/poolallocator.h index 0de13eaca..19bfbb6e7 100644 --- a/utils/common/poolallocator.h +++ b/utils/common/poolallocator.h @@ -25,6 +25,7 @@ #pragma once +#include #include #include #include @@ -32,9 +33,10 @@ #include #include +#include + #include -#include #include "countingallocator.h" @@ -57,6 +59,7 @@ class PoolAllocator , useLock(_useLock) , lock(false) { + std::cout << "PoolAllocator w/o counting allocator created" << std::endl; } PoolAllocator(allocators::CountingAllocator alloc, unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false, bool _useLock = false) @@ -69,6 +72,7 @@ class PoolAllocator , lock(false) , alloc(alloc) { + std::cout << "PoolAllocator with counting allocator created" << std::endl; } PoolAllocator(const PoolAllocator& p) : allocSize(p.allocSize) diff --git a/utils/common/stlpoolallocator.h b/utils/common/stlpoolallocator.h index 851d7af7d..3d511117d 100644 --- a/utils/common/stlpoolallocator.h +++ b/utils/common/stlpoolallocator.h @@ -101,11 +101,13 @@ STLPoolAllocator::STLPoolAllocator(joblist::ResourceManager* rm) { if (rm) { + std::cout << "STLPoolAllocator with RM " << std::endl; auto alloc = rm->getAllocator(); pa.reset(new PoolAllocator(alloc, DEFAULT_SIZE)); } else { + std::cout << "STLPoolAllocator w/o RM " << std::endl; pa.reset(new PoolAllocator(DEFAULT_SIZE)); } }