diff --git a/tests/counting_allocator.cpp b/tests/counting_allocator.cpp index 4fd2b71bc..b8e32809e 100644 --- a/tests/counting_allocator.cpp +++ b/tests/counting_allocator.cpp @@ -19,6 +19,7 @@ #include #include #include "countingallocator.h" +#include "rowgroup.h" using namespace allocators; @@ -108,6 +109,13 @@ TEST_F(CountingAllocatorTest, AllocateSharedUsesAllocator) ptr.reset(); EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); + + size_t allocSize = 16ULL * rowgroup::rgCommonSize; + auto buf = boost::allocate_shared(allocator, allocSize); + EXPECT_LE(allocatedMemory.load(), MemoryAllowance - allocSize); + + buf.reset(); + EXPECT_EQ(allocatedMemory.load(), MemoryAllowance); } // Test 5: Thread Safety - Concurrent Allocations and Deallocations diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 527f09709..4bd6194e1 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -179,17 +179,17 @@ void ByteStream::growBuf(BSSizeType toSize) } } -std::vector>& ByteStream::getLongStrings() +std::vector& ByteStream::getLongStrings() { return longStrings; } -const std::vector>& ByteStream::getLongStrings() const +const std::vector& ByteStream::getLongStrings() const { return longStrings; } -void ByteStream::setLongStrings(const std::vector>& other) +void ByteStream::setLongStrings(const std::vector& other) { longStrings = other; } diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 09da4b44a..8a1d4bd50 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -38,6 +38,7 @@ #include "any.hpp" #include "nullstring.h" #include "countingallocator.h" +#include "buffertypes.h" class ByteStreamTestSuite; @@ -447,9 +448,9 @@ class ByteStream : public Serializeable 3 * sizeof(uint32_t); // space for the BS magic & length & number of long strings. // Methods to get and set `long strings`. - EXPORT std::vector>& getLongStrings(); - EXPORT const std::vector>& getLongStrings() const; - EXPORT void setLongStrings(const std::vector>& other); + EXPORT std::vector& getLongStrings(); + EXPORT const std::vector& getLongStrings() const; + EXPORT void setLongStrings(const std::vector& other); friend class ::ByteStreamTestSuite; @@ -484,7 +485,7 @@ class ByteStream : public Serializeable BSBufType* fCurOutPtr; // the point in fBuf where data is extracted from next BSSizeType fMaxLen; // how big fBuf is currently // Stores `long strings`. - std::vector> longStrings; + std::vector longStrings; allocators::CountingAllocator* allocator = nullptr; }; diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index 07b43e8c3..998a00448 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -508,7 +508,7 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO return SBS(new ByteStream(0U)); res->advanceInputPtr(msglen); - std::vector> longStrings; + std::vector longStrings; try { for (uint32_t i = 0; i < longStringSize; ++i) @@ -520,7 +520,8 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO return SBS(new ByteStream(0U)); // Allocate new memory for the `long string`. - std::shared_ptr longString( + // WIP must account this allocation also. + rowgroup::StringStoreBufSPType longString( new uint8_t[sizeof(rowgroup::StringStore::MemChunk) + memChunk.currentSize]); uint8_t* longStringData = longString.get(); diff --git a/utils/rowgroup/buffertypes.h b/utils/rowgroup/buffertypes.h new file mode 100644 index 000000000..7d44c3f40 --- /dev/null +++ b/utils/rowgroup/buffertypes.h @@ -0,0 +1,30 @@ + +/* + Copyright (c) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ +#pragma once + +#include +#include + +namespace rowgroup +{ + using RGDataBufType = uint8_t[]; + using StringStoreBufType = uint8_t[]; + using StringStoreBufSPType = boost::shared_ptr; +} // namespace rowgroup diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 20aba90ae..db2c85aa9 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -96,19 +96,19 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) if (mem.size() > 0) lastMC = (MemChunk*)mem.back().get(); - std::cout << "StringStore::storeString len " << len << std::endl; + // std::cout << "StringStore::storeString len " << len << std::endl; if ((len + 4) >= CHUNK_SIZE) { auto allocSize = len + sizeof(MemChunk) + 4; - // if (alloc) - // { - // cout << "StringStore::storeString longStrings with alloc " << std::endl; - // longStrings.emplace_back(std::allocate_shared(*alloc, allocSize)); - // } - // else + if (alloc) { - cout << "StringStore::storeString longStrings no alloc " << std::endl; - longStrings.emplace_back(std::make_shared(allocSize)); + // cout << "StringStore::storeString longStrings with alloc " << std::endl; + longStrings.emplace_back(boost::allocate_shared(*alloc, allocSize)); + } + else + { + // cout << "StringStore::storeString longStrings no alloc " << std::endl; + longStrings.emplace_back(boost::make_shared(allocSize)); } // std::shared_ptr newOne(new uint8_t[len + sizeof(MemChunk) + 4]); lastMC = reinterpret_cast(longStrings.back().get()); @@ -127,13 +127,13 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) // if (lastMC) if (alloc) { - cout << "StringStore::storeString with alloc " << std::endl; + // cout << "StringStore::storeString with alloc " << std::endl; mem.emplace_back(boost::allocate_shared(*alloc, CHUNK_SIZE + sizeof(MemChunk))); // boost::allocate_shared) newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); } else { - cout << "StringStore::storeString no alloc " << std::endl; + // cout << "StringStore::storeString no alloc " << std::endl; mem.emplace_back(boost::make_shared(CHUNK_SIZE + sizeof(MemChunk))); // mem.emplace_back(boost::allocate_shared(*alloc, CHUNK_SIZE + sizeof(MemChunk))); // std::shared_ptr newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); @@ -196,7 +196,7 @@ void StringStore::deserialize(ByteStream& bs) // mem.clear(); bs >> count; - // mem.resize(count); + mem.reserve(count); bs >> tmp8; empty = (bool)tmp8; @@ -208,12 +208,12 @@ void StringStore::deserialize(ByteStream& bs) if (alloc) { - cout << "StringStore::deserialize with alloc " << std::endl; + // cout << "StringStore::deserialize with alloc " << std::endl; mem.emplace_back(boost::allocate_shared(*alloc, size + sizeof(MemChunk))); } else { - cout << "StringStore::deserialize no alloc " << std::endl; + // cout << "StringStore::deserialize no alloc " << std::endl; mem.emplace_back(boost::make_shared(size + sizeof(MemChunk))); } // mem[i].reset(new uint8_t[size + sizeof(MemChunk)]); @@ -231,7 +231,7 @@ void StringStore::deserialize(ByteStream& bs) void StringStore::clear() { vector > emptyv; - vector > emptyv2; + vector emptyv2; mem.swap(emptyv); longStrings.swap(emptyv2); empty = true; @@ -380,12 +380,12 @@ void RGData::reinit(const RowGroup& rg, uint32_t rowCount) { if (alloc) { - cout << "RGData::reinit with alloc " << std::endl; + // cout << "RGData::reinit with alloc " << std::endl; rowData = boost::allocate_shared(*alloc, rg.getDataSize(rowCount)); } else { - cout << "RGData::reinit no alloc " << std::endl; + // cout << "RGData::reinit no alloc " << std::endl; rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 665792f9d..d1cb58ccf 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -54,6 +54,7 @@ #include "collation.h" #include "common/hashfamily.h" +#include "buffertypes.h" #include #include "execinfo.h" @@ -131,9 +132,9 @@ inline T derefFromTwoVectorPtrs(const std::vector* outer, const std::vectoroperator[](outerIdx); } -using RGDataBufType = uint8_t[]; -// using RGDataBufType = std::vector; -using StringStoreBufType = uint8_t[]; +// using RGDataBufType = uint8_t[]; +// using StringStoreBufType = uint8_t[]; +// using StringStoreBufSPType = boost::shared_ptr; class StringStore { @@ -193,7 +194,7 @@ class StringStore std::vector> mem; // To store strings > 64KB (BLOB/TEXT) - std::vector> longStrings; + std::vector longStrings; bool empty = true; bool fUseStoreStringMutex = false; //@bug6065, make StringStore::storeString() thread safe boost::mutex fMutex;