fix(perf,allocator): adding unit test to check CountingAllocator behavior when it is wrapped in STLPoolAllocator

dbcon/joblist/resourcemanager.h

@@ -333,9 +333,14 @@ class ResourceManager
   }
   inline int64_t availableMemory() const
   {
-    return totalUmMemLimit.load(std::memory_order_relaxed);
+    return atomicops::atomicLoadRef(totalUmMemLimit);
   }
 
+  inline void setMemory(int64_t amount) 
+  {
+    atomicops::atomicStoreRef(totalUmMemLimit, amount);
+  }
+  
   /* old HJ mem interface, used by HashJoin */
   uint64_t getHjPmMaxMemorySmallSide(uint32_t sessionID)
   {

tests/CMakeLists.txt

@@ -88,6 +88,15 @@ if (WITH_UNITTESTS)
     target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
 
+    add_executable(stlpoolallocator stlpoolallocator.cpp)
+    target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
+    add_dependencies(stlpoolallocator googletest)
+    target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
+
+    add_executable(oid_server_return oid-server-return.cpp)
+    target_link_libraries(oid_server_return ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})
+
     add_executable(comparators_tests comparators-tests.cpp)
     target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
@@ -110,4 +119,3 @@ if (WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
     target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark)
     add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench)
 endif()
-

tests/poolallocator.cpp

@@ -202,7 +202,7 @@ TEST(PoolAllocatorTest, MultithreadedAllocationWithLock)
 
 static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024;
 
-// Test Fixture for AtomicCounterAllocator
+// Test Fixture for CounterAllocator
 class PoolallocatorTest : public ::testing::Test
 {
  protected:
@@ -226,30 +226,30 @@ class PoolallocatorTest : public ::testing::Test
 // Тест для проверки учёта потребления памяти в PoolAllocator.
 TEST_F(PoolallocatorTest, AllocationWithAccounting)
 {
-  int bufSize = 512;
+  int bufSize1 = 512;
   const unsigned CUSTOM_SIZE = 1024;
   PoolAllocator pa(allocator, CUSTOM_SIZE, false, true);
   EXPECT_EQ(pa.getWindowSize(), CUSTOM_SIZE);
   EXPECT_EQ(pa.getMemUsage(), 0ULL);
-  auto* ptr = pa.allocate(bufSize);
+  auto* ptr = pa.allocate(bufSize1);
 
   EXPECT_NE(ptr, nullptr);
-  EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
+  EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1);
   EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
   pa.deallocate(ptr);
   // B/c this PoolAllocator frees memory only when it's destroyed.
-  EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
+  EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize1);
   EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
 
-  bufSize = 64536;
+  int bufSize2 = 64536;
-  auto* ptr1 = pa.allocate(bufSize);
+  auto* ptr1 = pa.allocate(bufSize2);
 
   EXPECT_NE(ptr1, nullptr);
-  EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize);
+  EXPECT_LE(allocatedMemory.load(), MemoryAllowance - bufSize2 - CUSTOM_SIZE);
 
   pa.deallocate(ptr1);
   EXPECT_LE(allocatedMemory.load(), MemoryAllowance - CUSTOM_SIZE);
-  EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize);
+  EXPECT_GE(allocatedMemory.load(), MemoryAllowance - bufSize2);
 }
 
 TEST_F(PoolallocatorTest, MultithreadedAccountedAllocationWithLock)

tests/stlpoolallocator.cpp

@@ -0,0 +1,226 @@
+#include <gtest/gtest.h>
+#include <vector>
+#include <thread>
+#include <memory>
+#include "stlpoolallocator.h"
+#include "resourcemanager.h"
+
+using namespace utils;
+
+static const constexpr int64_t MemoryAllowance = 1 * 1024 * 1024;
+
+
+/**
+ * Test fixture for STLPoolAllocator tests
+ */
+class STLPoolAllocatorTest : public ::testing::Test
+{
+protected:
+    using TestType = int;  // Type we'll use for testing
+    using Allocator = STLPoolAllocator<TestType>;
+};
+
+/**
+ * Test default constructor and basic properties
+ */
+TEST_F(STLPoolAllocatorTest, DefaultConstructor)
+{
+    Allocator alloc;
+    EXPECT_EQ(alloc.getMemUsage(), 0ULL);
+    EXPECT_EQ(alloc.max_size(), std::numeric_limits<uint64_t>::max() / sizeof(TestType));
+}
+
+/**
+ * Test basic allocation and deallocation
+ */
+TEST_F(STLPoolAllocatorTest, BasicAllocation)
+{
+    Allocator alloc;
+    const size_t count = 10;
+    uint64_t initialUsage = alloc.getMemUsage();
+    
+    // Allocate memory for 10 integers
+    TestType* ptr = alloc.allocate(count);
+    ASSERT_NE(ptr, nullptr);
+    EXPECT_GT(alloc.getMemUsage(), initialUsage);
+    
+    // Construct objects
+    for (size_t i = 0; i < count; ++i)
+    {
+        alloc.construct(ptr + i, static_cast<TestType>(i));
+    }
+    
+    // Verify constructed values
+    for (size_t i = 0; i < count; ++i)
+    {
+        EXPECT_EQ(ptr[i], static_cast<TestType>(i));
+    }
+    
+    // Destroy objects
+    for (size_t i = 0; i < count; ++i)
+    {
+        alloc.destroy(ptr + i);
+    }
+    
+    // Deallocate memory
+    alloc.deallocate(ptr, count);
+}
+
+/**
+ * Test copy constructor and assignment operator
+ */
+TEST_F(STLPoolAllocatorTest, CopyAndAssignment)
+{
+    Allocator alloc1;
+    TestType* ptr = alloc1.allocate(1);
+    alloc1.construct(ptr, 42);
+    
+    // Test copy constructor
+    Allocator alloc2(alloc1);
+    EXPECT_EQ(alloc1.getMemUsage(), alloc2.getMemUsage());
+    
+    // Test assignment operator
+    Allocator alloc3;
+    alloc3 = alloc1;
+    EXPECT_EQ(alloc1.getMemUsage(), alloc3.getMemUsage());
+    
+    alloc1.destroy(ptr);
+    alloc1.deallocate(ptr, 1);
+}
+
+/**
+ * Test STL container integration (vector)
+ */
+TEST_F(STLPoolAllocatorTest, VectorIntegration)
+{
+    std::vector<TestType, Allocator> vec;
+    const size_t testSize = 1000;
+    
+    // Reserve to avoid reallocation
+    vec.reserve(testSize);
+    
+    // Add elements
+    for (size_t i = 0; i < testSize; ++i)
+    {
+        vec.push_back(static_cast<TestType>(i));
+    }
+    
+    // Verify elements
+    for (size_t i = 0; i < testSize; ++i)
+    {
+        EXPECT_EQ(vec[i], static_cast<TestType>(i));
+    }
+    
+    // Clear vector
+    vec.clear();
+    vec.shrink_to_fit();
+}
+
+// /**
+//  * Test multithreaded allocation
+//  */
+// TEST_F(STLPoolAllocatorTest, MultithreadedAllocation)
+// {
+//     const int THREAD_COUNT = 4;
+//     const size_t ALLOC_SIZE = 100;
+//     std::vector<std::thread> threads;
+    
+//     Allocator alloc;
+//     uint64_t initialUsage = alloc.getMemUsage();
+    
+//     // Create threads that will allocate memory
+//     for (int i = 0; i < THREAD_COUNT; ++i)
+//     {
+//         threads.emplace_back(
+//             [&alloc]()
+//             {
+//                 std::vector<TestType*, STLPoolAllocator<TestType*>> ptrs(alloc);
+                
+//                 for (size_t j = 0; j < 10; ++j)
+//                 {
+//                     TestType* ptr = alloc.allocate(ALLOC_SIZE);
+//                     for (size_t k = 0; k < ALLOC_SIZE; ++k)
+//                     {
+//                         alloc.construct(ptr + k, static_cast<TestType>(k));
+//                     }
+//                     ptrs.push_back(ptr);
+//                 }
+                
+//                 // Cleanup
+//                 for (auto ptr : ptrs)
+//                 {
+//                     for (size_t k = 0; k < ALLOC_SIZE; ++k)
+//                     {
+//                         alloc.destroy(ptr + k);
+//                     }
+//                     alloc.deallocate(ptr, ALLOC_SIZE);
+//                 }
+//             });
+//     }
+    
+//     // Wait for all threads to complete
+//     for (auto& th : threads)
+//     {
+//         th.join();
+//     }
+    
+//     // Memory usage should be greater than initial due to vector allocations
+//     EXPECT_GT(alloc.getMemUsage(), initialUsage);
+// }
+
+/**
+ * Test ResourceManager integration
+ */
+TEST_F(STLPoolAllocatorTest, ResourceManagerIntegration)
+{
+    joblist::ResourceManager rm(true, nullptr);
+    // To set the memory allowance
+    rm.setMemory(MemoryAllowance);
+    std::cout << "Memory allowance: " << MemoryAllowance << std::endl;
+
+    Allocator alloc(&rm);
+    
+    std::cout << "Memory available 1 : " << rm.availableMemory() << std::endl;
+    // Basic allocation test with ResourceManager
+    TestType* ptr = alloc.allocate(1);
+    ASSERT_NE(ptr, nullptr);
+    std::cout << "Memory available 2 : " << rm.availableMemory() << std::endl;
+    
+
+    alloc.construct(ptr, 42);
+    EXPECT_EQ(*ptr, 42);
+    
+    alloc.destroy(ptr);
+    alloc.deallocate(ptr, 1);
+    std::cout << "Memory available 3 : " << rm.availableMemory() << std::endl;
+
+    TestType* ptr2 = alloc.allocate(65537);
+    ASSERT_NE(ptr2, nullptr);
+    alloc.construct(ptr2, 42);
+    EXPECT_EQ(*ptr2, 42);
+    std::cout << "Memory available 4 : " << rm.availableMemory() << std::endl;
+
+    
+    alloc.destroy(ptr2);
+    alloc.deallocate(ptr2, 1);
+    std::cout << "Memory available 5 : " << rm.availableMemory() << std::endl;
+}
+
+/**
+ * Test rebind capability (required by STL)
+ */
+TEST_F(STLPoolAllocatorTest, RebindTest)
+{
+    using DoubleAllocator = typename Allocator::template rebind<double>::other;
+    DoubleAllocator doubleAlloc;
+    
+    // Allocate and construct a double
+    double* ptr = doubleAlloc.allocate(1);
+    ASSERT_NE(ptr, nullptr);
+    
+    doubleAlloc.construct(ptr, 3.14159);
+    EXPECT_DOUBLE_EQ(*ptr, 3.14159);
+    
+    doubleAlloc.destroy(ptr);
+    doubleAlloc.deallocate(ptr, 1);
+}

utils/common/atomicops.h

@@ -41,6 +41,16 @@ inline int64_t atomicSubRef(std::atomic<int64_t>& ref, int64_t val)
 {
   return ref.fetch_sub(val, std::memory_order_relaxed);
 }
+
+inline void atomicStoreRef(std::atomic<int64_t>& ref, int64_t val)
+{
+  ref.store(val, std::memory_order_relaxed);
+}
+
+inline int64_t atomicLoadRef(const std::atomic<int64_t>& ref)
+{
+  return ref.load(std::memory_order_relaxed);
+}
 // Returns the resulting, incremented value
 template <typename T>
 inline T atomicInc(volatile T* mem)

utils/common/poolallocator.cpp

@@ -25,6 +25,8 @@
 #include <boost/smart_ptr/allocate_shared_array.hpp>
 #include <boost/smart_ptr/make_shared_array.hpp>
 
+#include <iostream>
+
 #include "poolallocator.h"
 
 using namespace std;
@@ -34,6 +36,7 @@ namespace utils
 {
 PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v)
 {
+  std::cout << "PoolAllocator copy assignment" << std::endl;
   allocSize = v.allocSize;
   tmpSpace = v.tmpSpace;
   useLock = v.useLock;
@@ -54,15 +57,19 @@ void PoolAllocator::deallocateAll()
 void PoolAllocator::newBlock()
 {
   capacityRemaining = allocSize;
+  std::cout << "PoolAllocator new block" << std::endl;
 
   if (!tmpSpace || mem.size() == 0)
   {
     if (alloc)
     {
+      std::cout << "PoolAllocator new block with counting alloc" << std::endl;
+
       mem.emplace_back(boost::allocate_shared<PoolAllocatorBufType>(*alloc, allocSize)); 
     }
     else 
     {
+      std::cout << "PoolAllocator new block w/o counting alloc" << std::endl;
       mem.emplace_back(boost::make_shared<PoolAllocatorBufType>(allocSize));
     }
     nextAlloc = mem.back().get();
@@ -74,14 +81,17 @@ void PoolAllocator::newBlock()
 void* PoolAllocator::allocOOB(uint64_t size)
 {
   OOBMemInfo memInfo;
+  std::cout << "PoolAllocator allocOOB" << std::endl;
 
   memUsage += size;
   if (alloc)
   {
+    std::cout << "PoolAllocator allocOOB with counting alloc" << std::endl;
     memInfo.mem = boost::allocate_shared<PoolAllocatorBufType>(*alloc, size);
   }
   else 
   {
+    std::cout << "PoolAllocator allocOOB w/o counting alloc" << std::endl;
     memInfo.mem = boost::make_shared<PoolAllocatorBufType>(size);
   }
   memInfo.size = size;

utils/common/poolallocator.h

@@ -25,6 +25,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <unistd.h>
 #include <stdint.h>
 #include <optional>
@@ -32,9 +33,10 @@
 #include <map>
 #include <memory>
 
+#include <iostream>
+
 #include <boost/smart_ptr/allocate_shared_array.hpp>
 
-#include <atomic>
 
 #include "countingallocator.h"
 
@@ -57,6 +59,7 @@ class PoolAllocator
    , useLock(_useLock)
    , lock(false)
   {
+    std::cout << "PoolAllocator w/o counting allocator created" << std::endl;
   }
   PoolAllocator(allocators::CountingAllocator<PoolAllocatorBufType> alloc, unsigned windowSize = DEFAULT_WINDOW_SIZE,
                 bool isTmpSpace = false, bool _useLock = false)
@@ -69,6 +72,7 @@ class PoolAllocator
    , lock(false)
    , alloc(alloc)
   {
+    std::cout << "PoolAllocator with counting allocator created" << std::endl;
   }
   PoolAllocator(const PoolAllocator& p)
    : allocSize(p.allocSize)

utils/common/stlpoolallocator.h

@@ -101,11 +101,13 @@ STLPoolAllocator<T>::STLPoolAllocator(joblist::ResourceManager* rm)
 {
   if (rm) 
   {
+    std::cout << "STLPoolAllocator with RM " << std::endl;
     auto alloc = rm->getAllocator<PoolAllocatorBufType>();
     pa.reset(new PoolAllocator(alloc, DEFAULT_SIZE));
   }
   else
   {
+    std::cout << "STLPoolAllocator w/o RM " << std::endl;
     pa.reset(new PoolAllocator(DEFAULT_SIZE));
   }
 }