From fcdb6009215b3c2ff03749e40412e5f8d9659a6e Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat Date: Thu, 26 Sep 2024 12:30:13 +0530 Subject: [PATCH 4/6] Avoid translating RestrictInfo repeatedly RestrictInfo for the child relations (including the child join relations) are obtained by translating RestrictInfo applicable to the parent rel. Since these translations are not tracked, the same RestrictInfo may get translated multiple times for the same parent and child pair. When using partitionwise join this can happen as many times as the number of possible join orders between the partitioned tables and as many times a parent path is reparameterized for a child if a clause is used in such a path. Repeated translations are avoided by saving them in RestrictInfo hash table and reusing as needed. Ashutosh Bapat --- src/backend/optimizer/path/joinrels.c | 7 +- src/backend/optimizer/util/pathnode.c | 113 +++++++++++++++++-- src/backend/optimizer/util/relnode.c | 7 +- src/backend/optimizer/util/restrictinfo.c | 125 +++++++++++++++++++++- src/include/optimizer/restrictinfo.h | 7 ++ 5 files changed, 242 insertions(+), 17 deletions(-) diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 60d65762b5d..78d2f40472f 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -19,6 +19,7 @@ #include "optimizer/joininfo.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" +#include "optimizer/restrictinfo.h" #include "partitioning/partbounds.h" #include "utils/memutils.h" @@ -1652,10 +1653,8 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, * Construct restrictions applicable to the child join from those * applicable to the parent join. */ - child_restrictlist = - (List *) adjust_appendrel_attrs(root, - (Node *) parent_restrictlist, - nappinfos, appinfos); + child_restrictlist = get_child_restrictinfos(root, parent_restrictlist, + nappinfos, appinfos); /* Find or construct the child join's RelOptInfo */ child_joinrel = joinrel->part_rels[cnt_parts]; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 93e73cb44db..3e301ceeab1 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -26,6 +26,7 @@ #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/planmain.h" +#include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parsetree.h" #include "utils/memutils.h" @@ -4241,6 +4242,78 @@ reparameterize_path(PlannerInfo *root, Path *path, return NULL; } +/* + * build_child_iclauses_multilevel + * Translate IndexClause list applicable to the given top parent relation + * to that applicable to the given child relation where the child relation + * may be several levels below the top parent in the partition hierarchy. + * + * This is similar to adjust_appendrel_attrs_multilevel() for IndexClause + * except that it uses translated RestrictInfos if already available. + */ +static List * +build_child_iclauses_multilevel(PlannerInfo *root, + List *parent_iclauselist, + RelOptInfo *childrel, + RelOptInfo *top_parent) +{ + List *child_iclauses = NIL; + + foreach_node(IndexClause, iclause, parent_iclauselist) + { + List *rinfo_list = NIL; + List *child_rinfo_list; + IndexClause *child_iclause = makeNode(IndexClause); + ListCell *lcp; + ListCell *lcc; + List *indexquals; + + memcpy(child_iclause, iclause, sizeof(IndexClause)); + + /* + * Collect RestrictInfos to be translated. That's all there's to + * translate in an IndexClause. + */ + rinfo_list = lappend(rinfo_list, iclause->rinfo); + rinfo_list = list_concat(rinfo_list, iclause->indexquals); + + child_rinfo_list = get_child_restrictinfos_multilevel(root, rinfo_list, + childrel, top_parent); + child_iclause->rinfo = linitial(child_rinfo_list); + child_iclause->indexquals = NIL; + indexquals = list_delete_first(child_rinfo_list); + + /* + * indexquals of parent indexclause may have commuted RestrictInfos. + * Commute the child indexquals accordingly. + */ + forboth(lcc, indexquals, lcp, iclause->indexquals) + { + RestrictInfo *child_rinfo = lfirst_node(RestrictInfo, lcc); + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lcp); + Relids child_left_relids; + + child_left_relids = adjust_child_relids_multilevel(root, rinfo->left_relids, + childrel, top_parent); + if (!bms_equal(child_left_relids, child_rinfo->left_relids)) + { + OpExpr *clause = castNode(OpExpr, rinfo->clause); + + Assert(bms_equal(child_left_relids, child_rinfo->right_relids)); + + child_rinfo = commute_restrictinfo(child_rinfo, clause->opno); + } + + child_iclause->indexquals = lappend(child_iclause->indexquals, child_rinfo); + } + + list_free(rinfo_list); + child_iclauses = lappend(child_iclauses, child_iclause); + } + + return child_iclauses; +} + /* * reparameterize_path_by_child * Given a path parameterized by the parent of the given child relation, @@ -4343,7 +4416,10 @@ do { \ IndexPath *ipath = (IndexPath *) path; ADJUST_CHILD_ATTRS(ipath->indexinfo->indrestrictinfo); - ADJUST_CHILD_ATTRS(ipath->indexclauses); + ipath->indexclauses = + build_child_iclauses_multilevel(root, + ipath->indexclauses, + child_rel, child_rel->top_parent); new_path = (Path *) ipath; } break; @@ -4422,7 +4498,11 @@ do { \ REPARAMETERIZE_CHILD_PATH(jpath->outerjoinpath); REPARAMETERIZE_CHILD_PATH(jpath->innerjoinpath); - ADJUST_CHILD_ATTRS(jpath->joinrestrictinfo); + jpath->joinrestrictinfo = + get_child_restrictinfos_multilevel(root, + jpath->joinrestrictinfo, + child_rel, + child_rel->top_parent); new_path = (Path *) npath; } break; @@ -4434,8 +4514,16 @@ do { \ REPARAMETERIZE_CHILD_PATH(jpath->outerjoinpath); REPARAMETERIZE_CHILD_PATH(jpath->innerjoinpath); - ADJUST_CHILD_ATTRS(jpath->joinrestrictinfo); - ADJUST_CHILD_ATTRS(mpath->path_mergeclauses); + jpath->joinrestrictinfo = + get_child_restrictinfos_multilevel(root, + jpath->joinrestrictinfo, + child_rel, + child_rel->top_parent); + mpath->path_mergeclauses = + get_child_restrictinfos_multilevel(root, + mpath->path_mergeclauses, + child_rel, + child_rel->top_parent); new_path = (Path *) mpath; } break; @@ -4447,8 +4535,16 @@ do { \ REPARAMETERIZE_CHILD_PATH(jpath->outerjoinpath); REPARAMETERIZE_CHILD_PATH(jpath->innerjoinpath); - ADJUST_CHILD_ATTRS(jpath->joinrestrictinfo); - ADJUST_CHILD_ATTRS(hpath->path_hashclauses); + jpath->joinrestrictinfo = + get_child_restrictinfos_multilevel(root, + jpath->joinrestrictinfo, + child_rel, + child_rel->top_parent); + hpath->path_hashclauses = + get_child_restrictinfos_multilevel(root, + hpath->path_hashclauses, + child_rel, + child_rel->top_parent); new_path = (Path *) hpath; } break; @@ -4525,7 +4621,10 @@ do { \ new_ppi->ppi_req_outer = bms_copy(required_outer); new_ppi->ppi_rows = old_ppi->ppi_rows; new_ppi->ppi_clauses = old_ppi->ppi_clauses; - ADJUST_CHILD_ATTRS(new_ppi->ppi_clauses); + new_ppi->ppi_clauses = + get_child_restrictinfos_multilevel(root, + new_ppi->ppi_clauses, child_rel, + child_rel->top_parent); new_ppi->ppi_serials = bms_copy(old_ppi->ppi_serials); rel->ppilist = lappend(rel->ppilist, new_ppi); diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index ff507331a06..740b6e33631 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -961,10 +961,9 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, nappinfos, appinfos); /* Construct joininfo list. */ - joinrel->joininfo = (List *) adjust_appendrel_attrs(root, - (Node *) parent_joinrel->joininfo, - nappinfos, - appinfos); + joinrel->joininfo = get_child_restrictinfos(root, + parent_joinrel->joininfo, + nappinfos, appinfos); /* * Lateral relids referred in child join will be same as that referred in diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 9279062a877..11584cdb4ec 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -14,12 +14,13 @@ */ #include "postgres.h" +#include "common/hashfn.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "optimizer/appendinfo.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" -#include "common/hashfn.h" static Expr *make_sub_restrictinfos(PlannerInfo *root, Expr *clause, @@ -779,7 +780,7 @@ get_child_rinfo_hash(PlannerInfo *root) * add_restrictinfo * Add the given RestrictInfo to the RestrictInfo hash table. */ -extern void +void add_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo) { HTAB *rinfo_hash = get_child_rinfo_hash(root); @@ -791,6 +792,13 @@ add_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo) key.required_relids = rinfo->required_relids; rinfo_entry = hash_search(rinfo_hash, &key, HASH_ENTER, &found); + /* + * If the given RestrictInfo is already present in the hash table, + * multiple instances of the same RestrictInfo may have been created. This + * function is a good place to flag that. While multiple instances of same + * RestrictInfo being created is not a correctness issue, they consume + * memory unnecessarily. + */ Assert(!found); rinfo_entry->rinfo = rinfo; } @@ -817,3 +825,116 @@ find_restrictinfo(PlannerInfo *root, int rinfo_serial, Relids required_relids) NULL); return (rinfo_entry ? rinfo_entry->rinfo : NULL); } + +/* + * get_child_restrictinfos + * Returns list of child RestrictInfos obtained by translating the given + * parent RestrictInfos according to the given AppendRelInfos. + * + * RestrictInfos applicable to a child relation are obtained by translating the + * corresponding RestrictInfos applicable to the parent relation. The same + * parent RestictInfo appears in restrictlist or joininfo list of many parent + * join relations and thus may get translated multiple times producing multiple + * instances of the same child RestrictInfo. In order to avoid that we store the + * translated child RestrictInfos in a hash table and reused. + * + * If a required translated RestrictInfo is available in the RestrictInfo hash + * table, the function includes the available child RestrictInfo to the result + * list. Otherwise, it translates the corresponding parent RestrictInfo and + * adds it to the RestrictInfo hash table and the result list. + */ +List * +get_child_restrictinfos(PlannerInfo *root, List *parent_restrictinfos, + int nappinfos, AppendRelInfo **appinfos) +{ + List *child_clauselist = NIL; + + foreach_node(RestrictInfo, parent_rinfo, parent_restrictinfos) + { + Relids child_req_relids; + RestrictInfo *child_rinfo = NULL; + + child_req_relids = adjust_child_relids(parent_rinfo->required_relids, + nappinfos, appinfos); + + if (bms_equal(child_req_relids, parent_rinfo->required_relids)) + { + /* + * If no relid was translated, child's RestrictInfo is same as + * that of parent. + */ + child_rinfo = parent_rinfo; + } + else + { + child_rinfo = find_restrictinfo(root, parent_rinfo->rinfo_serial, child_req_relids); + + /* + * This function may be called thousands of times when there are + * thousands of partitions involved. We won't require the + * translated child relids further. Hence free those to avoid + * accumulating huge amounts of memory. + */ + bms_free(child_req_relids); + } + + if (!child_rinfo) + { + child_rinfo = castNode(RestrictInfo, + adjust_appendrel_attrs(root, (Node *) parent_rinfo, + nappinfos, appinfos)); + add_restrictinfo(root, child_rinfo); + } + + child_clauselist = lappend(child_clauselist, child_rinfo); + } + + return child_clauselist; +} + +/* + * get_child_restrictinfos_multilevel + * Similar to get_child_restrictinfos() but for translations through multiple + * levels of partitioning hierarchy. + * + * This function is similar to adjust_appendrel_attrs_multilevel() except that + * this function makes use of RestrictInfo hash table. + */ +List * +get_child_restrictinfos_multilevel(PlannerInfo *root, List *parent_clauselist, + RelOptInfo *child_rel, RelOptInfo *top_parent) +{ + AppendRelInfo **appinfos; + int nappinfos; + List *tmp_clauselist = parent_clauselist; + List *child_clauselist; + + /* Recursively traverse up the partition hierarchy. */ + if (child_rel->parent != top_parent) + { + if (child_rel->parent) + { + tmp_clauselist = get_child_restrictinfos_multilevel(root, + parent_clauselist, + child_rel->parent, + top_parent); + } + else + elog(ERROR, "child_rel is not a child of top_parent"); + } + + appinfos = find_appinfos_by_relids(root, child_rel->relids, &nappinfos); + child_clauselist = get_child_restrictinfos(root, tmp_clauselist, + nappinfos, appinfos); + + /* + * This function will be called thousands of times, if there are thousands + * of partitions involved. Free temporary objects created in this function + * to avoid accumulating huge memory. + */ + pfree(appinfos); + if (tmp_clauselist != parent_clauselist) + list_free(tmp_clauselist); + + return child_clauselist; +} diff --git a/src/include/optimizer/restrictinfo.h b/src/include/optimizer/restrictinfo.h index 57d2210ce95..5e1f0136495 100644 --- a/src/include/optimizer/restrictinfo.h +++ b/src/include/optimizer/restrictinfo.h @@ -61,6 +61,13 @@ extern bool join_clause_is_movable_into(RestrictInfo *rinfo, extern RestrictInfo *find_restrictinfo(PlannerInfo *root, int rinfo_serial, Relids child_required_relids); extern void add_restrictinfo(PlannerInfo *root, RestrictInfo *child_rinfo); +extern List *get_child_restrictinfos(PlannerInfo *root, + List *parent_restrictinfos, + int nappinfos, AppendRelInfo **appinfos); +extern List *get_child_restrictinfos_multilevel(PlannerInfo *root, + List *parent_clauselist, + RelOptInfo *child_rel, + RelOptInfo *top_parent); /* * clause_sides_match_join -- 2.34.1