database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-07 00:36:50 +03:00
Code Activity

Reduce memory used by partitionwise joins

Browse Source 
In try_partitionwise_join, we aim to break down the join between two
partitioned relations into joins between matching partitions.  To
achieve this, we iterate through each pair of partitions from the two
joining relations and create child-join relations for them.  With
potentially thousands of partitions, the local objects allocated in
each iteration can accumulate significant memory usage.  Therefore, we
opt to eagerly free these local objects at the end of each iteration.

In line with this approach, this patch frees the bitmap set that
represents the relids of child-join relations at the end of each
iteration.  Additionally, it modifies build_child_join_rel() to reuse
the AppendRelInfo structures generated within each iteration.

Author: Ashutosh Bapat
Reviewed-by: David Christensen, Richard Guo
Discussion: https://postgr.es/m/CAExHW5s4EqY43oB=ne6B2=-xLgrs9ZGeTr1NXwkGFt2j-OmaQQ@mail.gmail.com
This commit is contained in:
Richard Guo
2024-07-29 11:35:51 +09:00
parent f47b33a191
commit 513f4472a4
3 changed files with 16 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/backend/optimizer/path/joinrels.c
View File

@ -1547,6 +1547,7 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
RelOptInfo *child_joinrel;
AppendRelInfo **appinfos;
int nappinfos;
Relids child_relids;
if (joinrel->partbounds_merged)
{
@ -1642,9 +1643,8 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
child_rel2->relids);
/* Find the AppendRelInfo structures */
appinfos = find_appinfos_by_relids(root,
bms_union(child_rel1->relids,
child_rel2->relids),
child_relids = bms_union(child_rel1->relids, child_rel2->relids);
appinfos = find_appinfos_by_relids(root, child_relids,
&nappinfos);
/*
@ -1662,7 +1662,7 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
{
child_joinrel = build_child_join_rel(root, child_rel1, child_rel2,
joinrel, child_restrictlist,
child_sjinfo);
child_sjinfo, nappinfos, appinfos);
joinrel->part_rels[cnt_parts] = child_joinrel;
joinrel->live_parts = bms_add_member(joinrel->live_parts, cnt_parts);
joinrel->all_partrels = bms_add_members(joinrel->all_partrels,
@ -1679,7 +1679,14 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
child_joinrel, child_sjinfo,
child_restrictlist);
/*
* When there are thousands of partitions involved, this loop will
* accumulate a significant amount of memory usage from objects that
* are only needed within the loop. Free these local objects eagerly
* at the end of each iteration.
*/
pfree(appinfos);
bms_free(child_relids);
free_child_join_sjinfo(child_sjinfo);
}
}

18
src/backend/optimizer/util/relnode.c
View File

@ -876,15 +876,15 @@ build_join_rel(PlannerInfo *root,
* 'restrictlist': list of RestrictInfo nodes that apply to this particular
* pair of joinable relations
* 'sjinfo': child join's join-type details
* 'nappinfos' and 'appinfos': AppendRelInfo array for child relids
*/
RelOptInfo *
build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel,
RelOptInfo *inner_rel, RelOptInfo *parent_joinrel,
List *restrictlist, SpecialJoinInfo *sjinfo)
List *restrictlist, SpecialJoinInfo *sjinfo,
int nappinfos, AppendRelInfo **appinfos)
{
RelOptInfo *joinrel = makeNode(RelOptInfo);
AppendRelInfo **appinfos;
int nappinfos;
/* Only joins between "other" relations land here. */
Assert(IS_OTHER_REL(outer_rel) && IS_OTHER_REL(inner_rel));
@ -892,16 +892,6 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel,
/* The parent joinrel should have consider_partitionwise_join set. */
Assert(parent_joinrel->consider_partitionwise_join);
/*
* Find the AppendRelInfo structures for the child baserels. We'll need
* these for computing the child join's relid set, and later for mapping
* Vars to the child rel.
*/
appinfos = find_appinfos_by_relids(root,
bms_union(outer_rel->relids,
inner_rel->relids),
&nappinfos);
joinrel->reloptkind = RELOPT_OTHER_JOINREL;
joinrel->relids = adjust_child_relids(parent_joinrel->relids,
nappinfos, appinfos);
@ -1017,8 +1007,6 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel,
nappinfos, appinfos,
parent_joinrel, joinrel);
pfree(appinfos);
return joinrel;
}