database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-5522 Properly process pm join result count. (#2909)

Browse Source 
This patch:
1. Properly processes situation when pm join result count is exceeded.
2. Adds session variable 'columnstore_max_pm_join_result_count` to control the limit.
This commit is contained in:
Denis Khalikov
2023-08-04 16:55:45 +03:00
committed by GitHub
parent 4f580d109d
commit 896e8dd769
17 changed files with 144 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
dbcon/execplan/calpontselectexecutionplan.cpp
View File

@ -89,6 +89,7 @@ CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(const int location)
, fDJSPartitionSize(100 * 1024 * 1024)
, fDJSMaxPartitionTreeDepth(8)
, fDJSForceRun(false)
, fMaxPmJoinResultCount(1048576)
, // 100MB mem usage for disk based join,
fUMMemLimit(numeric_limits<int64_t>::max())
, fIsDML(false)
@ -463,6 +464,7 @@ void CalpontSelectExecutionPlan::serialize(messageqcpp::ByteStream& b) const
b << fDJSPartitionSize;
b << fDJSMaxPartitionTreeDepth;
b << (uint8_t)fDJSForceRun;
b << (uint32_t)fMaxPmJoinResultCount;
b << fUMMemLimit;
b << (uint8_t)fIsDML;
messageqcpp::ByteStream::octbyte timeZone = fTimeZone;
@ -661,6 +663,7 @@ void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b)
b >> fDJSPartitionSize;
b >> fDJSMaxPartitionTreeDepth;
b >> (uint8_t&)fDJSForceRun;
b >> (uint32_t&)fMaxPmJoinResultCount;
b >> fUMMemLimit;
b >> tmp8;
fIsDML = tmp8;

10
dbcon/execplan/calpontselectexecutionplan.h
View File

@ -708,6 +708,15 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan
return fDJSForceRun;
}
void maxPmJoinResultCount(uint32_t value)
{
fMaxPmJoinResultCount = value;
}
uint32_t maxPmJoinResultCount()
{
return fMaxPmJoinResultCount;
}
void umMemLimit(uint64_t l)
{
fUMMemLimit = l;
@ -942,6 +951,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan
uint64_t fDJSPartitionSize = 100 * 1024 * 1024;
uint32_t fDJSMaxPartitionTreeDepth = 8;
bool fDJSForceRun = false;
uint32_t fMaxPmJoinResultCount = 1048576;
int64_t fUMMemLimit = numeric_limits<int64_t>::max();
bool fIsDML = false;
long fTimeZone = 0;

1
dbcon/joblist/batchprimitiveprocessor-jl.cpp
View File

@ -1105,6 +1105,7 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const
/* if HAS_JOINER, send the init params */
if (flags & HAS_JOINER)
{
bs << (uint32_t)maxPmJoinResultCount;
if (ot == ROW_GROUP)
{
idbassert(tJoiners.size() > 0);

6
dbcon/joblist/batchprimitiveprocessor-jl.h
View File

@ -252,6 +252,11 @@ class BatchPrimitiveProcessorJL
uuid = u;
}
void setMaxPmJoinResultCount(uint32_t count)
{
maxPmJoinResultCount = count;
}
private:
const size_t perColumnProjectWeight_ = 10;
const size_t perColumnFilteringWeight_ = 10;
@ -374,6 +379,7 @@ class BatchPrimitiveProcessorJL
unsigned fJoinerChunkSize;
uint32_t dbRoot;
bool hasSmallOuterJoin;
uint32_t maxPmJoinResultCount = 1048576;
uint32_t _priority;

2
dbcon/joblist/jlf_common.h
View File

@ -211,6 +211,7 @@ struct JobInfo
, wfqLimitStart(0)
, wfqLimitCount(-1)
, timeZone(0)
, maxPmJoinResultCount(1048576)
{
}
ResourceManager* rm;
@ -368,6 +369,7 @@ struct JobInfo
bool djsForceRun;
bool isDML;
long timeZone;
uint32_t maxPmJoinResultCount;
// This is for tracking any dynamically allocated ParseTree objects
// in simpleScalarFilterToParseTree() for later deletion in

1
dbcon/joblist/joblistfactory.cpp
View File

@ -2068,6 +2068,7 @@ SJLP makeJobList_(CalpontExecutionPlan* cplan, ResourceManager* rm,
jobInfo.partitionSize = csep->djsPartitionSize();
jobInfo.djsMaxPartitionTreeDepth = csep->djsMaxPartitionTreeDepth();
jobInfo.djsForceRun = csep->djsForceRun();
jobInfo.maxPmJoinResultCount = csep->maxPmJoinResultCount();
jobInfo.umMemLimit.reset(new int64_t);
*(jobInfo.umMemLimit) = csep->umMemLimit();
jobInfo.isDML = csep->isDML();

1
dbcon/joblist/jobstep.cpp
View File

@ -91,6 +91,7 @@ JobStep::JobStep(const JobInfo& j)
, fProgress(0)
, fStartTime(-1)
, fTimeZone(j.timeZone)
, fMaxPmJoinResultCount(j.maxPmJoinResultCount)
{
QueryTeleServerParms tsp;
string teleServerHost(Config::makeConfig()->getConfig("QueryTele", "Host"));

1
dbcon/joblist/jobstep.h
View File

@ -497,6 +497,7 @@ class JobStep
int64_t fStartTime;
int64_t fLastStepTeleTime;
long fTimeZone;
uint32_t fMaxPmJoinResultCount;
private:
static boost::mutex fLogMutex;

7
dbcon/joblist/primitivestep.h
View File

@ -1198,6 +1198,11 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep
return bRunFEonPM;
}
void setMaxPmJoinResultCount(uint32_t count)
{
maxPmJoinResultCount = count;
}
protected:
void sendError(uint16_t status);
@ -1339,6 +1344,8 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep
boost::shared_ptr<RowGroupDL> deliveryDL;
uint32_t deliveryIt;
uint32_t maxPmJoinResultCount;
class JoinLocalData
{
public:

4
dbcon/joblist/tuple-bps.cpp
View File

@ -1458,9 +1458,13 @@ void TupleBPS::run()
fBPP->setThreadCount(fMaxNumProcessorThreads);
if (doJoin)
{
for (i = 0; i < smallSideCount; i++)
tjoiners[i]->setThreadCount(fMaxNumProcessorThreads);
fBPP->setMaxPmJoinResultCount(fMaxPmJoinResultCount);
}
if (fe1)
fBPP->setFEGroup1(fe1, fe1Input);

1
dbcon/mysql/ha_mcs_execplan.cpp
View File

@ -6642,6 +6642,7 @@ void setExecutionParams(gp_walk_info& gwi, SCSEP& csep)
csep->djsPartitionSize(get_diskjoin_bucketsize(gwi.thd) * 1024ULL * 1024);
csep->djsMaxPartitionTreeDepth(get_diskjoin_max_partition_tree_depth(gwi.thd));
csep->djsForceRun(get_diskjoin_force_run(gwi.thd));
csep->maxPmJoinResultCount(get_max_pm_join_result_count(gwi.thd));
if (get_um_mem_limit(gwi.thd) == 0)
csep->umMemLimit(numeric_limits<int64_t>::max());
else

14
dbcon/mysql/ha_mcs_sysvars.cpp
View File

@ -141,6 +141,10 @@ static MYSQL_THDVAR_ULONG(diskjoin_max_partition_tree_depth, PLUGIN_VAR_RQCMDARG
static MYSQL_THDVAR_BOOL(diskjoin_force_run, PLUGIN_VAR_RQCMDARG, "Force run for the disk join step.", NULL,
NULL, 0);
static MYSQL_THDVAR_ULONG(max_pm_join_result_count, PLUGIN_VAR_RQCMDARG,
"The maximum size of the join result for the single block on BPP.", NULL, NULL,
1048576, 1, ~0U, 1);
static MYSQL_THDVAR_ULONG(um_mem_limit, PLUGIN_VAR_RQCMDARG,
"Per user Memory limit(MB). Switch to disk-based JOIN when limit is reached", NULL,
NULL, 0, 0, ~0U, 1);
@ -233,6 +237,7 @@ st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type),
MYSQL_SYSVAR(diskjoin_bucketsize),
MYSQL_SYSVAR(diskjoin_max_partition_tree_depth),
MYSQL_SYSVAR(diskjoin_force_run),
MYSQL_SYSVAR(max_pm_join_result_count),
MYSQL_SYSVAR(um_mem_limit),
MYSQL_SYSVAR(double_for_decimal_math),
MYSQL_SYSVAR(decimal_overflow_check),
@ -448,6 +453,15 @@ void set_diskjoin_force_run(THD* thd, bool value)
THDVAR(thd, diskjoin_force_run) = value;
}
ulong get_max_pm_join_result_count(THD* thd)
{
return (thd == NULL) ? 0 : THDVAR(thd, max_pm_join_result_count);
}
void set_max_pm_join_result_count(THD* thd, ulong value)
{
THDVAR(thd, max_pm_join_result_count) = value;
}
ulong get_um_mem_limit(THD* thd)
{
return (thd == NULL) ? 0 : THDVAR(thd, um_mem_limit);

3
dbcon/mysql/ha_mcs_sysvars.h
View File

@ -114,6 +114,9 @@ void set_diskjoin_force_run(THD* thd, bool value);
ulong get_diskjoin_max_partition_tree_depth(THD* thd);
void set_diskjoin_max_partition_tree_depth(THD* thd, ulong value);
ulong get_max_pm_join_result_count(THD* thd);
void set_max_pm_join_result_count(THD* thd, ulong value);
ulong get_um_mem_limit(THD* thd);
void set_um_mem_limit(THD* thd, ulong value);