database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Randomly start a new generation if the free memory is less than 30%

Browse Source
This commit is contained in:
Alexey Antipovsky
2022-08-22 20:22:15 +03:00
committed by Roman Nozdrin
parent 7251f58818
commit 15ce531270
2 changed files with 24 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
utils/rowgroup/rowstorage.cpp
View File

@ -352,6 +352,11 @@ class MemManager
return std::numeric_limits<int64_t>::max();
}
virtual int64_t getConfigured() const
{
return std::numeric_limits<int64_t>::max();
}
virtual bool isStrict() const
{
return false;
@ -399,6 +404,11 @@ class RMMemManager : public MemManager
fMemUsed = 0;
}
int64_t getConfigured() const final
{
return fRm->getConfiguredUMMemLimit();
}
int64_t getFree() const final
{
return std::min(fRm->availableMemory(), *fSessLimit);
@ -1537,6 +1547,9 @@ RowAggStorage::RowAggStorage(const std::string& tmpDir, RowGroup* rowGroupOut, R
, fTmpDir(tmpDir)
, fRowGroupOut(rowGroupOut)
, fKeysRowGroup(keysRowGroup)
, fRD()
, fRandGen(fRD())
, fRandDistr(0, 100)
{
char suffix[PATH_MAX];
snprintf(suffix, sizeof(suffix), "/p%u-t%p/", getpid(), this);
@ -1722,6 +1735,7 @@ void RowAggStorage::dump()
break;
}
int64_t totalMem = fMM->getConfigured();
// If the generations are allowed and there are less than half of
// rowgroups in memory, then we start a new generation
if (fAllowGenerations && fStorage->fLRU->size() < fStorage->fRGDatas.size() / 2 &&
@ -1729,6 +1743,12 @@ void RowAggStorage::dump()
{
startNewGeneration();
}
else if (fAllowGenerations &&
freeMem < totalMem / 10 * 3 &&
fRandDistr(fRandGen) < 30)
{
startNewGeneration();
}
else if (!fAllowGenerations && freeMem < 0 && freeAttempts == 1)
{
// safety guard so aggregation couldn't eat all available memory

4
utils/rowgroup/rowstorage.h
View File

@ -20,6 +20,7 @@
#include "resourcemanager.h"
#include "rowgroup.h"
#include "idbcompress.h"
#include <random>
#include <sys/stat.h>
#include <unistd.h>
@ -367,6 +368,9 @@ class RowAggStorage
bool fInitialized{false};
rowgroup::RowGroup* fRowGroupOut;
rowgroup::RowGroup* fKeysRowGroup;
std::random_device fRD;
std::mt19937 fRandGen;
std::uniform_int_distribution<uint8_t> fRandDistr;
};
} // namespace rowgroup