Randomly start a new generation if the free memory is less than 30%

utils/rowgroup/rowstorage.cpp

@@ -352,6 +352,11 @@ class MemManager
     return std::numeric_limits<int64_t>::max();
   }
 
+  virtual int64_t getConfigured() const
+  {
+    return std::numeric_limits<int64_t>::max();
+  }
+
   virtual bool isStrict() const
   {
     return false;
@@ -399,6 +404,11 @@ class RMMemManager : public MemManager
     fMemUsed = 0;
   }
 
+  int64_t getConfigured() const final
+  {
+    return fRm->getConfiguredUMMemLimit();
+  }
+
   int64_t getFree() const final
   {
     return std::min(fRm->availableMemory(), *fSessLimit);
@@ -1537,6 +1547,9 @@ RowAggStorage::RowAggStorage(const std::string& tmpDir, RowGroup* rowGroupOut, R
  , fTmpDir(tmpDir)
  , fRowGroupOut(rowGroupOut)
  , fKeysRowGroup(keysRowGroup)
+ , fRD()
+ , fRandGen(fRD())
+ , fRandDistr(0, 100)
 {
   char suffix[PATH_MAX];
   snprintf(suffix, sizeof(suffix), "/p%u-t%p/", getpid(), this);
@@ -1722,6 +1735,7 @@ void RowAggStorage::dump()
       break;
   }
 
+  int64_t totalMem = fMM->getConfigured();
   // If the generations are allowed and there are less than half of
   // rowgroups in memory, then we start a new generation
   if (fAllowGenerations && fStorage->fLRU->size() < fStorage->fRGDatas.size() / 2 &&
@@ -1729,6 +1743,12 @@ void RowAggStorage::dump()
   {
     startNewGeneration();
   }
+  else if (fAllowGenerations &&
+           freeMem < totalMem / 10 * 3 &&
+           fRandDistr(fRandGen) < 30)
+  {
+    startNewGeneration();
+  }
   else if (!fAllowGenerations && freeMem < 0 && freeAttempts == 1)
   {
     // safety guard so aggregation couldn't eat all available memory

utils/rowgroup/rowstorage.h

@@ -20,6 +20,7 @@
 #include "resourcemanager.h"
 #include "rowgroup.h"
 #include "idbcompress.h"
+#include <random>
 #include <sys/stat.h>
 #include <unistd.h>
 
@@ -367,6 +368,9 @@ class RowAggStorage
   bool fInitialized{false};
   rowgroup::RowGroup* fRowGroupOut;
   rowgroup::RowGroup* fKeysRowGroup;
+  std::random_device fRD;
+  std::mt19937 fRandGen;
+  std::uniform_int_distribution<uint8_t> fRandDistr;
 };
 
 }  // namespace rowgroup