From 1be82f859b809be424d53cc00ecb27f62a13f9ea Mon Sep 17 00:00:00 2001 From: Alexey Antipovsky Date: Mon, 22 Aug 2022 20:22:15 +0300 Subject: [PATCH] Randomly start a new generation if the free memory is less than 30% --- utils/rowgroup/rowstorage.cpp | 20 ++++++++++++++++++++ utils/rowgroup/rowstorage.h | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/utils/rowgroup/rowstorage.cpp b/utils/rowgroup/rowstorage.cpp index bfa60dcea..558ba2759 100644 --- a/utils/rowgroup/rowstorage.cpp +++ b/utils/rowgroup/rowstorage.cpp @@ -301,6 +301,11 @@ class MemManager return std::numeric_limits::max(); } + virtual int64_t getConfigured() const + { + return std::numeric_limits::max(); + } + virtual bool isStrict() const { return false; @@ -348,6 +353,11 @@ class RMMemManager : public MemManager fMemUsed = 0; } + int64_t getConfigured() const final + { + return fRm->getConfiguredUMMemLimit(); + } + int64_t getFree() const final { return std::min(fRm->availableMemory(), *fSessLimit); @@ -1486,6 +1496,9 @@ RowAggStorage::RowAggStorage(const std::string& tmpDir, RowGroup* rowGroupOut, R , fTmpDir(tmpDir) , fRowGroupOut(rowGroupOut) , fKeysRowGroup(keysRowGroup) + , fRD() + , fRandGen(fRD()) + , fRandDistr(0, 100) { char suffix[PATH_MAX]; snprintf(suffix, sizeof(suffix), "/p%u-t%p/", getpid(), this); @@ -1671,6 +1684,7 @@ void RowAggStorage::dump() break; } + int64_t totalMem = fMM->getConfigured(); // If the generations are allowed and there are less than half of // rowgroups in memory, then we start a new generation if (fAllowGenerations && fStorage->fLRU->size() < fStorage->fRGDatas.size() / 2 && @@ -1678,6 +1692,12 @@ void RowAggStorage::dump() { startNewGeneration(); } + else if (fAllowGenerations && + freeMem < totalMem / 10 * 3 && + fRandDistr(fRandGen) < 30) + { + startNewGeneration(); + } else if (!fAllowGenerations && freeMem < 0 && freeAttempts == 1) { // safety guard so aggregation couldn't eat all available memory diff --git a/utils/rowgroup/rowstorage.h b/utils/rowgroup/rowstorage.h index 58ba3cfb8..20886bb16 100644 --- a/utils/rowgroup/rowstorage.h +++ b/utils/rowgroup/rowstorage.h @@ -21,6 +21,7 @@ #include "resourcemanager.h" #include "rowgroup.h" #include "idbcompress.h" +#include #include #include @@ -368,6 +369,9 @@ class RowAggStorage bool fInitialized{false}; rowgroup::RowGroup* fRowGroupOut; rowgroup::RowGroup* fKeysRowGroup; + std::random_device fRD; + std::mt19937 fRandGen; + std::uniform_int_distribution fRandDistr; }; } // namespace rowgroup