From ed4de69e7ae180eca380ae581152b6650175661f Mon Sep 17 00:00:00 2001 From: amitlan Date: Wed, 22 Dec 2021 16:55:17 +0900 Subject: [PATCH v1] Teach AcquireExecutorLocks() to acquire fewer locks in some cases Currently, AcquireExecutorLocks() loops over the range table of a given PlannedStmt and locks all relations found therein, even those that won't actually be scanned during execution due to being eliminated by "initial" pruning that is applied during the initialization of their owning Append or MergeAppend node. This makes AcquireExecutorLocks() itself do the "initial" pruning on nodes that support it and lock only those relations that are contained in the subnodes that survive the pruning. To that end, AcquireExecutorLocks() now loops over a bitmapset of RT indexes, those of the RTEs of "lockable" relations, instead of the whole range table to find such entries. When pruning is possible, the bitmapset is constructed by walking the plan tree to locate nodes that allow "initial" (or "pre-execution") pruning and disregarding relations from subnodes that don't survive the pruning instructions. PlannedStmt gets a bitmapset field to store the RT indexes of lockable relations that is populated when contructing the flat range table in setrefs.c. It is used as is in the absence of any prunable nodes. PlannedStmt also gets a new field that indicates whether any of the nodes of the plan tree contain "initial" (or "pre-execution") pruning steps, which saves the trouble of walking the plan tree only to find whether that's the case. ExecFindInitialMatchingSubPlans() is refactored to allow being called outside a full-fledged executor context. --- src/backend/executor/execParallel.c | 2 + src/backend/executor/execPartition.c | 534 ++++++++++++++++++------- src/backend/executor/nodeAppend.c | 39 +- src/backend/executor/nodeMergeAppend.c | 39 +- src/backend/nodes/copyfuncs.c | 4 + src/backend/nodes/nodeFuncs.c | 121 +++++- src/backend/nodes/outfuncs.c | 5 + src/backend/nodes/readfuncs.c | 4 + src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/plan/setrefs.c | 10 + src/backend/partitioning/partprune.c | 57 ++- src/backend/utils/cache/plancache.c | 217 +++++++++- src/include/executor/execPartition.h | 13 +- src/include/nodes/nodeFuncs.h | 3 + src/include/nodes/pathnodes.h | 6 + src/include/nodes/plannodes.h | 15 + src/include/partitioning/partprune.h | 3 + 17 files changed, 866 insertions(+), 208 deletions(-) diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index f8a4a40e7b..d14e60724b 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -182,8 +182,10 @@ ExecSerializePlan(Plan *plan, EState *estate) pstmt->transientPlan = false; pstmt->dependsOnRole = false; pstmt->parallelModeNeeded = false; + pstmt->usesPreExecPruning = false; pstmt->planTree = plan; pstmt->rtable = estate->es_range_table; + pstmt->relationRTIs = NULL; pstmt->resultRelations = NIL; pstmt->appendRelations = NIL; diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc54e..8c63272398 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -24,6 +24,7 @@ #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "parser/parsetree.h" #include "partitioning/partbounds.h" #include "partitioning/partdesc.h" #include "partitioning/partprune.h" @@ -186,7 +187,8 @@ static void ExecInitPruningContext(PartitionPruneContext *context, List *pruning_steps, PartitionDesc partdesc, PartitionKey partkey, - PlanState *planstate); + PlanState *planstate, + ExprContext *econtext); static void find_matching_subplans_recurse(PartitionPruningData *prunedata, PartitionedRelPruningData *pprune, bool initial_prune, @@ -1511,8 +1513,7 @@ adjust_partition_colnos(List *colnos, ResultRelInfo *leaf_part_rri) /* * ExecCreatePartitionPruneState - * Build the data structure required for calling - * ExecFindInitialMatchingSubPlans and ExecFindMatchingSubPlans. + * Build the data structure for run-time pruning * * 'planstate' is the parent plan node's execution state. * @@ -1526,10 +1527,20 @@ adjust_partition_colnos(List *colnos, ResultRelInfo *leaf_part_rri) * as children. The data stored in each PartitionedRelPruningData can be * re-used each time we re-evaluate which partitions match the pruning steps * provided in each PartitionedRelPruneInfo. + * + * This does not consider initial_pruning_steps because they must already have + * been performed by the caller and the subplans remaining after doing so are + * given as 'initially_valid_subplans'. The translation data to be put into + * PartitionPruneState that allows conversion of partition indexes into subplan + * indexes are updated here to account for the unneeded subplans having been + * removed by initial pruning. 'nsubplans' gives the number of subplans that + * were present before initial pruning. */ PartitionPruneState * ExecCreatePartitionPruneState(PlanState *planstate, - PartitionPruneInfo *partitionpruneinfo) + PartitionPruneInfo *partitionpruneinfo, + Bitmapset *initially_valid_subplans, + int nsubplans) { EState *estate = planstate->state; PartitionPruneState *prunestate; @@ -1537,6 +1548,15 @@ ExecCreatePartitionPruneState(PlanState *planstate, ListCell *lc; int i; + /* + * Only create a PartitionPruneState if pruning needs to be performed + * during the execution of the owning plan. Note that this means the + * initial pruning steps which are used to determine the set of subplans + * that are valid for actual execution are performed without creating a + * PartitionPruneState; see ExecFindInitialMatchingSubPlans(). + */ + Assert(partitionpruneinfo->contains_exec_steps); + /* For data reading, executor always omits detached partitions */ if (estate->es_partition_directory == NULL) estate->es_partition_directory = @@ -1555,7 +1575,6 @@ ExecCreatePartitionPruneState(PlanState *planstate, prunestate->execparamids = NULL; /* other_subplans can change at runtime, so we need our own copy */ prunestate->other_subplans = bms_copy(partitionpruneinfo->other_subplans); - prunestate->do_initial_prune = false; /* may be set below */ prunestate->do_exec_prune = false; /* may be set below */ prunestate->num_partprunedata = n_part_hierarchies; @@ -1702,23 +1721,17 @@ ExecCreatePartitionPruneState(PlanState *planstate, pprune->present_parts = bms_copy(pinfo->present_parts); /* - * Initialize pruning contexts as needed. + * Initialize pruning contexts as needed, ignoring any + * initial_pruning_steps because they must already have been + * performed. */ - pprune->initial_pruning_steps = pinfo->initial_pruning_steps; - if (pinfo->initial_pruning_steps) - { - ExecInitPruningContext(&pprune->initial_context, - pinfo->initial_pruning_steps, - partdesc, partkey, planstate); - /* Record whether initial pruning is needed at any level */ - prunestate->do_initial_prune = true; - } pprune->exec_pruning_steps = pinfo->exec_pruning_steps; if (pinfo->exec_pruning_steps) { ExecInitPruningContext(&pprune->exec_context, pinfo->exec_pruning_steps, - partdesc, partkey, planstate); + partdesc, partkey, planstate, + planstate->ps_ExprContext); /* Record whether exec pruning is needed at any level */ prunestate->do_exec_prune = true; } @@ -1735,18 +1748,136 @@ ExecCreatePartitionPruneState(PlanState *planstate, i++; } + /* + * If exec-time pruning is required and subplans appear to have been + * pruned by initial pruning steps, then we must re-sequence the subplan + * indexes so that ExecFindMatchingSubPlans() properly returns the indexes + * of the subplans that have remained after initial pruning, that is, + * initially_valid_subplans. + * + * We can safely skip this when !do_exec_prune, even though that leaves + * invalid data in pruneinfo, because that data won't be consulted again + * (cf initial Assert in ExecFindMatchingSubPlans). + */ + if (prunestate->do_exec_prune && + bms_num_members(initially_valid_subplans) < nsubplans) + { + int *new_subplan_indexes; + Bitmapset *new_other_subplans; + int i; + int newidx; + + /* + * First we must build a temporary array which maps old subplan + * indexes to new ones. For convenience of initialization, we use + * 1-based indexes in this array and leave pruned items as 0. + */ + new_subplan_indexes = (int *) palloc0(sizeof(int) * nsubplans); + newidx = 1; + i = -1; + while ((i = bms_next_member(initially_valid_subplans, i)) >= 0) + { + Assert(i < nsubplans); + new_subplan_indexes[i] = newidx++; + } + + /* + * Now we can update each PartitionedRelPruneInfo's subplan_map with + * new subplan indexes. We must also recompute its present_parts + * bitmap. + */ + for (i = 0; i < prunestate->num_partprunedata; i++) + { + PartitionPruningData *prunedata = prunestate->partprunedata[i]; + int j; + + /* + * Within each hierarchy, we perform this loop in back-to-front + * order so that we determine present_parts for the lowest-level + * partitioned tables first. This way we can tell whether a + * sub-partitioned table's partitions were entirely pruned so we + * can exclude it from the current level's present_parts. + */ + for (j = prunedata->num_partrelprunedata - 1; j >= 0; j--) + { + PartitionedRelPruningData *pprune = &prunedata->partrelprunedata[j]; + int nparts = pprune->nparts; + int k; + + /* We just rebuild present_parts from scratch */ + bms_free(pprune->present_parts); + pprune->present_parts = NULL; + + for (k = 0; k < nparts; k++) + { + int oldidx = pprune->subplan_map[k]; + int subidx; + + /* + * If this partition existed as a subplan then change the + * old subplan index to the new subplan index. The new + * index may become -1 if the partition was pruned above, + * or it may just come earlier in the subplan list due to + * some subplans being removed earlier in the list. If + * it's a subpartition, add it to present_parts unless + * it's entirely pruned. + */ + if (oldidx >= 0) + { + Assert(oldidx < nsubplans); + pprune->subplan_map[k] = new_subplan_indexes[oldidx] - 1; + + if (new_subplan_indexes[oldidx] > 0) + pprune->present_parts = + bms_add_member(pprune->present_parts, k); + } + else if ((subidx = pprune->subpart_map[k]) >= 0) + { + PartitionedRelPruningData *subprune; + + subprune = &prunedata->partrelprunedata[subidx]; + + if (!bms_is_empty(subprune->present_parts)) + pprune->present_parts = + bms_add_member(pprune->present_parts, k); + } + } + } + } + + /* + * We must also recompute the other_subplans set, since indexes in it + * may change. + */ + new_other_subplans = NULL; + i = -1; + while ((i = bms_next_member(prunestate->other_subplans, i)) >= 0) + new_other_subplans = bms_add_member(new_other_subplans, + new_subplan_indexes[i] - 1); + + bms_free(prunestate->other_subplans); + prunestate->other_subplans = new_other_subplans; + + pfree(new_subplan_indexes); + } + return prunestate; } /* * Initialize a PartitionPruneContext for the given list of pruning steps. + * + * At least one of 'planstate' or 'econtext' must be passed to be able to + * successfully evaluate any non-Const expressions contained in the + * steps. */ static void ExecInitPruningContext(PartitionPruneContext *context, List *pruning_steps, PartitionDesc partdesc, PartitionKey partkey, - PlanState *planstate) + PlanState *planstate, + ExprContext *econtext) { int n_steps; int partnatts; @@ -1767,6 +1898,7 @@ ExecInitPruningContext(PartitionPruneContext *context, context->ppccontext = CurrentMemoryContext; context->planstate = planstate; + context->exprcontext = econtext; /* Initialize expression state for each expression we need */ context->exprstates = (ExprState **) @@ -1795,8 +1927,13 @@ ExecInitPruningContext(PartitionPruneContext *context, step->step.step_id, keyno); - context->exprstates[stateidx] = - ExecInitExpr(expr, context->planstate); + if (planstate == NULL) + context->exprstates[stateidx] = + ExecInitExprWithParams(expr, + econtext->ecxt_param_list_info); + else + context->exprstates[stateidx] = + ExecInitExpr(expr, context->planstate); } keyno++; } @@ -1809,171 +1946,283 @@ ExecInitPruningContext(PartitionPruneContext *context, * pruning, disregarding any pruning constraints involving PARAM_EXEC * Params. * - * If additional pruning passes will be required (because of PARAM_EXEC - * Params), we must also update the translation data that allows conversion - * of partition indexes into subplan indexes to account for the unneeded - * subplans having been removed. + * Must only be called once per 'pruneinfo', and only if initial pruning is + * required. * - * Must only be called once per 'prunestate', and only if initial pruning - * is required. + * 'param' contains information about any EXTERN parameters that might be + * present in the initial pruning steps. * - * 'nsubplans' must be passed as the total number of unpruned subplans. + * The RT indexes of unpruned parents are returned in *parentrelids if asked + * for by the caller. */ Bitmapset * -ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, int nsubplans) +ExecFindInitialMatchingSubPlans(PartitionPruneInfo *pruneinfo, + EState *estate, List *rtable, + ParamListInfo params, + Bitmapset **parentrelids) { Bitmapset *result = NULL; MemoryContext oldcontext; + MemoryContext tmpcontext; int i; + ListCell *lc; + int n_part_hierarchies; + bool free_estate = false; + ExprContext *econtext; + PartitionPruningData **partprunedata; + PartitionDirectory pdir; - /* Caller error if we get here without do_initial_prune */ - Assert(prunestate->do_initial_prune); - - /* - * Switch to a temp context to avoid leaking memory in the executor's - * query-lifespan memory context. - */ - oldcontext = MemoryContextSwitchTo(prunestate->prune_context); - - /* - * For each hierarchy, do the pruning tests, and add nondeletable - * subplans' indexes to "result". - */ - for (i = 0; i < prunestate->num_partprunedata; i++) - { - PartitionPruningData *prunedata; - PartitionedRelPruningData *pprune; + /* Caller error if we get here without contains_init_steps */ + Assert(pruneinfo->contains_init_steps); - prunedata = prunestate->partprunedata[i]; - pprune = &prunedata->partrelprunedata[0]; - /* Perform pruning without using PARAM_EXEC Params */ - find_matching_subplans_recurse(prunedata, pprune, true, &result); + if (parentrelids) + *parentrelids = NULL; - /* Expression eval may have used space in node's ps_ExprContext too */ - if (pprune->initial_pruning_steps) - ResetExprContext(pprune->initial_context.planstate->ps_ExprContext); + /* Set up EState if not in the executor proper. */ + if (estate == NULL) + { + estate = CreateExecutorState(); + estate->es_param_list_info = params; + free_estate = true; } - /* Add in any subplans that partition pruning didn't account for */ - result = bms_add_members(result, prunestate->other_subplans); - - MemoryContextSwitchTo(oldcontext); + /* An ExprContext to evaluate expressions. */ + econtext = CreateExprContext(estate); - /* Copy result out of the temp context before we reset it */ - result = bms_copy(result); + /* PartitionDirectory, creating one if not there already. */ + pdir = estate->es_partition_directory; + if (pdir == NULL) + { + /* Omits detached partitions, just like in the executor proper. */ + pdir = CreatePartitionDirectory(CurrentMemoryContext, false); + estate->es_partition_directory = pdir; + } - MemoryContextReset(prunestate->prune_context); + /* A temporary context to allocate stuff needded to run pruning steps. */ + tmpcontext = AllocSetContextCreate(CurrentMemoryContext, + "initial pruning working data", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(tmpcontext); /* - * If exec-time pruning is required and we pruned subplans above, then we - * must re-sequence the subplan indexes so that ExecFindMatchingSubPlans - * properly returns the indexes from the subplans which will remain after - * execution of this function. + * Stuff that follows matches exactly what ExecCreatePartitionPruneState() + * does, except we don't need a PartitionPruneState here, so don't call + * that function. * - * We can safely skip this when !do_exec_prune, even though that leaves - * invalid data in prunestate, because that data won't be consulted again - * (cf initial Assert in ExecFindMatchingSubPlans). + * XXX some refactoring might be good. */ - if (prunestate->do_exec_prune && bms_num_members(result) < nsubplans) + + /* PartitionPruningData for each partition hierarachy. */ + n_part_hierarchies = list_length(pruneinfo->prune_infos); + Assert(n_part_hierarchies > 0); + partprunedata = (PartitionPruningData **) + palloc(sizeof(PartitionPruningData *) * n_part_hierarchies); + i = 0; + foreach(lc, pruneinfo->prune_infos) { - int *new_subplan_indexes; - Bitmapset *new_other_subplans; - int i; - int newidx; + PartitionPruningData *prunedata; + List *partrelpruneinfos = lfirst_node(List, lc); + int npartrelpruneinfos = list_length(partrelpruneinfos); + ListCell *lc2; + int j; - /* - * First we must build a temporary array which maps old subplan - * indexes to new ones. For convenience of initialization, we use - * 1-based indexes in this array and leave pruned items as 0. - */ - new_subplan_indexes = (int *) palloc0(sizeof(int) * nsubplans); - newidx = 1; - i = -1; - while ((i = bms_next_member(result, i)) >= 0) - { - Assert(i < nsubplans); - new_subplan_indexes[i] = newidx++; - } + /* PartitionedRelPruningData per parent in the hierarchy. */ + prunedata = (PartitionPruningData *) + palloc(offsetof(PartitionPruningData, partrelprunedata) + + npartrelpruneinfos * sizeof(PartitionedRelPruningData)); + partprunedata[i] = prunedata; + prunedata->num_partrelprunedata = npartrelpruneinfos; - /* - * Now we can update each PartitionedRelPruneInfo's subplan_map with - * new subplan indexes. We must also recompute its present_parts - * bitmap. - */ - for (i = 0; i < prunestate->num_partprunedata; i++) + j = 0; + foreach(lc2, partrelpruneinfos) { - PartitionPruningData *prunedata = prunestate->partprunedata[i]; - int j; + PartitionedRelPruneInfo *pinfo = lfirst_node(PartitionedRelPruneInfo, lc2); + PartitionedRelPruningData *pprune = &prunedata->partrelprunedata[j]; + RangeTblEntry *partrte = rt_fetch(pinfo->rtindex, rtable); + Relation partrel; + PartitionDesc partdesc; + PartitionKey partkey; /* - * Within each hierarchy, we perform this loop in back-to-front - * order so that we determine present_parts for the lowest-level - * partitioned tables first. This way we can tell whether a - * sub-partitioned table's partitions were entirely pruned so we - * can exclude it from the current level's present_parts. + * We can rely on the copies of the partitioned table's partition + * key and partition descriptor appearing in its relcache entry, + * because that entry will be held open and locked while the + * PartitionedRelPruningData is in use. */ - for (j = prunedata->num_partrelprunedata - 1; j >= 0; j--) + partrel = table_open(partrte->relid, partrte->rellockmode); + partkey = RelationGetPartitionKey(partrel); + partdesc = PartitionDirectoryLookup(pdir, partrel); + + /* + * Initialize the subplan_map and subpart_map. + * + * Because we request detached partitions to be included, and + * detaching waits for old transactions, it is safe to assume that + * no partitions have disappeared since this query was planned. + * + * However, new partitions may have been added. + */ + Assert(partdesc->nparts >= pinfo->nparts); + pprune->nparts = partdesc->nparts; + pprune->subplan_map = palloc(sizeof(int) * partdesc->nparts); + if (partdesc->nparts == pinfo->nparts) { - PartitionedRelPruningData *pprune = &prunedata->partrelprunedata[j]; - int nparts = pprune->nparts; - int k; + /* + * There are no new partitions, so this is simple. We can + * simply point to the subpart_map from the plan, but we must + * copy the subplan_map since we may change it later. + */ + pprune->subpart_map = pinfo->subpart_map; + memcpy(pprune->subplan_map, pinfo->subplan_map, + sizeof(int) * pinfo->nparts); - /* We just rebuild present_parts from scratch */ - bms_free(pprune->present_parts); - pprune->present_parts = NULL; + /* + * Double-check that the list of unpruned relations has not + * changed. (Pruned partitions are not in relid_map[].) + */ +#ifdef USE_ASSERT_CHECKING + for (int k = 0; k < pinfo->nparts; k++) + { + Assert(partdesc->oids[k] == pinfo->relid_map[k] || + pinfo->subplan_map[k] == -1); + } +#endif + } + else + { + int pd_idx = 0; + int pp_idx; - for (k = 0; k < nparts; k++) + /* + * Some new partitions have appeared since plan time, and + * those are reflected in our PartitionDesc but were not + * present in the one used to construct subplan_map and + * subpart_map. So we must construct new and longer arrays + * where the partitions that were originally present map to + * the same sub-structures, and any added partitions map to + * -1, as if the new partitions had been pruned. + * + * Note: pinfo->relid_map[] may contain InvalidOid entries for + * partitions pruned by the planner. We cannot tell exactly + * which of the partdesc entries these correspond to, but we + * don't have to; just skip over them. The non-pruned + * relid_map entries, however, had better be a subset of the + * partdesc entries and in the same order. + */ + pprune->subpart_map = palloc(sizeof(int) * partdesc->nparts); + for (pp_idx = 0; pp_idx < partdesc->nparts; pp_idx++) { - int oldidx = pprune->subplan_map[k]; - int subidx; + /* Skip any InvalidOid relid_map entries */ + while (pd_idx < pinfo->nparts && + !OidIsValid(pinfo->relid_map[pd_idx])) + pd_idx++; - /* - * If this partition existed as a subplan then change the - * old subplan index to the new subplan index. The new - * index may become -1 if the partition was pruned above, - * or it may just come earlier in the subplan list due to - * some subplans being removed earlier in the list. If - * it's a subpartition, add it to present_parts unless - * it's entirely pruned. - */ - if (oldidx >= 0) + if (pd_idx < pinfo->nparts && + pinfo->relid_map[pd_idx] == partdesc->oids[pp_idx]) { - Assert(oldidx < nsubplans); - pprune->subplan_map[k] = new_subplan_indexes[oldidx] - 1; - - if (new_subplan_indexes[oldidx] > 0) - pprune->present_parts = - bms_add_member(pprune->present_parts, k); + /* match... */ + pprune->subplan_map[pp_idx] = + pinfo->subplan_map[pd_idx]; + pprune->subpart_map[pp_idx] = + pinfo->subpart_map[pd_idx]; + pd_idx++; } - else if ((subidx = pprune->subpart_map[k]) >= 0) + else { - PartitionedRelPruningData *subprune; - - subprune = &prunedata->partrelprunedata[subidx]; - - if (!bms_is_empty(subprune->present_parts)) - pprune->present_parts = - bms_add_member(pprune->present_parts, k); + /* this partdesc entry is not in the plan */ + pprune->subplan_map[pp_idx] = -1; + pprune->subpart_map[pp_idx] = -1; } } + + /* + * It might seem that we need to skip any trailing InvalidOid + * entries in pinfo->relid_map before checking that we scanned + * all of the relid_map. But we will have skipped them above, + * because they must correspond to some partdesc->oids + * entries; we just couldn't tell which. + */ + if (pd_idx != pinfo->nparts) + elog(ERROR, "could not match partition child tables to plan elements"); } + + /* present_parts is also subject to later modification */ + pprune->present_parts = bms_copy(pinfo->present_parts); + pprune->initial_pruning_steps = pinfo->initial_pruning_steps; + if (pprune->initial_pruning_steps) + ExecInitPruningContext(&pprune->initial_context, + pprune->initial_pruning_steps, + partdesc, partkey, NULL, econtext); + + table_close(partrel, NoLock); + j++; } + i++; + } + + /* + * For each hierarchy, do the pruning tests, and add nondeletable + * subplans' indexes to result. + */ + for (i = 0; i < n_part_hierarchies; i++) + { + PartitionPruningData *prunedata = partprunedata[i]; + PartitionedRelPruningData *pprune; /* - * We must also recompute the other_subplans set, since indexes in it - * may change. + * We pass the 1st item belonging to the root table of the hierarchy + * and find_matching_subplans_recurse() takes care of recursing to + * other (lower-level) parents as needed. */ - new_other_subplans = NULL; - i = -1; - while ((i = bms_next_member(prunestate->other_subplans, i)) >= 0) - new_other_subplans = bms_add_member(new_other_subplans, - new_subplan_indexes[i] - 1); + pprune = &prunedata->partrelprunedata[0]; + find_matching_subplans_recurse(prunedata, pprune, true, &result); - bms_free(prunestate->other_subplans); - prunestate->other_subplans = new_other_subplans; + /* + * Collect the RT indexes of surviving parents if the callers asked + * to see them. + */ + if (parentrelids) + { + int j; + List *partrelpruneinfos = list_nth_node(List, + pruneinfo->prune_infos, + i); - pfree(new_subplan_indexes); + for (j = 0; j < prunedata->num_partrelprunedata; j++) + { + PartitionedRelPruneInfo *pinfo = list_nth_node(PartitionedRelPruneInfo, + partrelpruneinfos, j); + + pprune = &prunedata->partrelprunedata[j]; + if (!bms_is_empty(pprune->present_parts)) + *parentrelids = bms_add_member(*parentrelids, pinfo->rtindex); + } + } + + /* Release space used up in our ExprContext. */ + ResetExprContext(econtext); + } + + /* Add in any subplans that partition pruning didn't account for. */ + result = bms_add_members(result, pruneinfo->other_subplans); + + MemoryContextSwitchTo(oldcontext); + + /* Copy result out of the temp context before we reset it */ + result = bms_copy(result); + if (parentrelids) + *parentrelids = bms_copy(*parentrelids); + + /* Safe to drop the temporary context */ + MemoryContextDelete(tmpcontext); + + /* Free the ExprState, and EState if needed. */ + FreeExprContext(econtext, true); + if (free_estate) + { + FreeExecutorState(estate); + estate = NULL; } return result; @@ -2018,6 +2267,11 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate) prunedata = prunestate->partprunedata[i]; pprune = &prunedata->partrelprunedata[0]; + /* + * We pass the 1st item belonging to the root table of the hierarchy + * and find_matching_subplans_recurse() takes care of recursing to + * other (lower-level) parents as needed. + */ find_matching_subplans_recurse(prunedata, pprune, false, &result); /* Expression eval may have used space in node's ps_ExprContext too */ diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 6a2daa6e76..7f813476ab 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -136,24 +136,15 @@ ExecInitAppend(Append *node, EState *estate, int eflags) /* If run-time partition pruning is enabled, then set that up now */ if (node->part_prune_info != NULL) { - PartitionPruneState *prunestate; - - /* We may need an expression context to evaluate partition exprs */ - ExecAssignExprContext(estate, &appendstate->ps); - - /* Create the working data structure for pruning. */ - prunestate = ExecCreatePartitionPruneState(&appendstate->ps, - node->part_prune_info); - appendstate->as_prune_state = prunestate; - - /* Perform an initial partition prune, if required. */ - if (prunestate->do_initial_prune) + if (node->part_prune_info->contains_init_steps) { - /* Determine which subplans survive initial pruning */ - validsubplans = ExecFindInitialMatchingSubPlans(prunestate, - list_length(node->appendplans)); - + validsubplans = + ExecFindInitialMatchingSubPlans(node->part_prune_info, + estate, estate->es_range_table, + estate->es_param_list_info, + NULL); nplans = bms_num_members(validsubplans); + Assert(nplans >= 0); } else { @@ -163,12 +154,26 @@ ExecInitAppend(Append *node, EState *estate, int eflags) validsubplans = bms_add_range(NULL, 0, nplans - 1); } + /* Create the working data structure for run-time pruning. */ + if (node->part_prune_info->contains_exec_steps) + { + PartitionPruneState *prunestate; + + /* We may need an expression context to evaluate partition exprs */ + ExecAssignExprContext(estate, &appendstate->ps); + prunestate = ExecCreatePartitionPruneState(&appendstate->ps, + node->part_prune_info, + validsubplans, + list_length(node->appendplans)); + + appendstate->as_prune_state = prunestate; + } /* * When no run-time pruning is required and there's at least one * subplan, we can fill as_valid_subplans immediately, preventing * later calls to ExecFindMatchingSubPlans. */ - if (!prunestate->do_exec_prune && nplans > 0) + else appendstate->as_valid_subplans = bms_add_range(NULL, 0, nplans - 1); } else diff --git a/src/backend/executor/nodeMergeAppend.c b/src/backend/executor/nodeMergeAppend.c index 617bffb206..51c5c3433d 100644 --- a/src/backend/executor/nodeMergeAppend.c +++ b/src/backend/executor/nodeMergeAppend.c @@ -84,23 +84,15 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) /* If run-time partition pruning is enabled, then set that up now */ if (node->part_prune_info != NULL) { - PartitionPruneState *prunestate; - - /* We may need an expression context to evaluate partition exprs */ - ExecAssignExprContext(estate, &mergestate->ps); - - prunestate = ExecCreatePartitionPruneState(&mergestate->ps, - node->part_prune_info); - mergestate->ms_prune_state = prunestate; - - /* Perform an initial partition prune, if required. */ - if (prunestate->do_initial_prune) + if (node->part_prune_info->contains_init_steps) { - /* Determine which subplans survive initial pruning */ - validsubplans = ExecFindInitialMatchingSubPlans(prunestate, - list_length(node->mergeplans)); - + validsubplans = + ExecFindInitialMatchingSubPlans(node->part_prune_info, + estate, estate->es_range_table, + estate->es_param_list_info, + NULL); nplans = bms_num_members(validsubplans); + Assert(nplans >= 0); } else { @@ -110,13 +102,28 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) validsubplans = bms_add_range(NULL, 0, nplans - 1); } + /* Create the working data structure for run-time pruning. */ + if (node->part_prune_info->contains_exec_steps) + { + PartitionPruneState *prunestate; + + /* We may need an expression context to evaluate partition exprs */ + ExecAssignExprContext(estate, &mergestate->ps); + prunestate = ExecCreatePartitionPruneState(&mergestate->ps, + node->part_prune_info, + validsubplans, + list_length(node->mergeplans)); + + mergestate->ms_prune_state = prunestate; + } /* * When no run-time pruning is required and there's at least one * subplan, we can fill as_valid_subplans immediately, preventing * later calls to ExecFindMatchingSubPlans. */ - if (!prunestate->do_exec_prune && nplans > 0) + else mergestate->ms_valid_subplans = bms_add_range(NULL, 0, nplans - 1); + } else { diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index df0b747883..57f2fce3d4 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -94,9 +94,11 @@ _copyPlannedStmt(const PlannedStmt *from) COPY_SCALAR_FIELD(transientPlan); COPY_SCALAR_FIELD(dependsOnRole); COPY_SCALAR_FIELD(parallelModeNeeded); + COPY_SCALAR_FIELD(usesPreExecPruning); COPY_SCALAR_FIELD(jitFlags); COPY_NODE_FIELD(planTree); COPY_NODE_FIELD(rtable); + COPY_BITMAPSET_FIELD(relationRTIs); COPY_NODE_FIELD(resultRelations); COPY_NODE_FIELD(appendRelations); COPY_NODE_FIELD(subplans); @@ -1277,6 +1279,8 @@ _copyPartitionPruneInfo(const PartitionPruneInfo *from) PartitionPruneInfo *newnode = makeNode(PartitionPruneInfo); COPY_NODE_FIELD(prune_infos); + COPY_SCALAR_FIELD(contains_init_steps); + COPY_SCALAR_FIELD(contains_exec_steps); COPY_BITMAPSET_FIELD(other_subplans); return newnode; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index e276264882..a13ee087a8 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -31,7 +31,10 @@ static bool planstate_walk_subplans(List *plans, bool (*walker) (), void *context); static bool planstate_walk_members(PlanState **planstates, int nplans, bool (*walker) (), void *context); - +static bool plan_walk_subplans(List *plans, + bool (*walker) (), + void *context); +static bool plan_walk_members(List *plans, bool (*walker) (), void *context); /* * exprType - @@ -4105,3 +4108,119 @@ planstate_walk_members(PlanState **planstates, int nplans, return false; } + +/* + * plan_tree_walker --- walk plantrees + * + * The walker has already visited the current node, and so we need only + * recurse into any sub-nodes it has. + */ +bool +plan_tree_walker(Plan *plan, + bool (*walker) (), + void *context) +{ + ListCell *lc; + + /* Guard against stack overflow due to overly complex plan trees */ + check_stack_depth(); + + /* initPlan-s */ + if (plan_walk_subplans(plan->initPlan, walker, context)) + return true; + + /* lefttree */ + if (outerPlan(plan)) + { + if (walker(outerPlan(plan), context)) + return true; + } + + /* righttree */ + if (innerPlan(plan)) + { + if (walker(innerPlan(plan), context)) + return true; + } + + /* special child plans */ + switch (nodeTag(plan)) + { + case T_Append: + if (plan_walk_members(((Append *) plan)->appendplans, + walker, context)) + return true; + break; + case T_MergeAppend: + if (plan_walk_members(((MergeAppend *) plan)->mergeplans, + walker, context)) + return true; + break; + case T_BitmapAnd: + if (plan_walk_members(((BitmapAnd *) plan)->bitmapplans, + walker, context)) + return true; + break; + case T_BitmapOr: + if (plan_walk_members(((BitmapOr *) plan)->bitmapplans, + walker, context)) + return true; + break; + case T_SubqueryScan: + if (walker(((SubqueryScan *) plan)->subplan, context)) + return true; + break; + case T_CustomScan: + foreach(lc, ((CustomScan *) plan)->custom_plans) + { + if (walker((Plan *) lfirst(lc), context)) + return true; + } + break; + default: + break; + } + + return false; +} + +/* + * Walk a list of SubPlans (or initPlans, which also use SubPlan nodes). + */ +static bool +plan_walk_subplans(List *plans, + bool (*walker) (), + void *context) +{ + ListCell *lc; + PlannedStmt *plannedstmt = (PlannedStmt *) context; + + foreach(lc, plans) + { + SubPlan *sp = lfirst_node(SubPlan, lc); + Plan *p = list_nth(plannedstmt->subplans, sp->plan_id - 1); + + if (walker(p, context)) + return true; + } + + return false; +} + +/* + * Walk the constituent plans of a ModifyTable, Append, MergeAppend, + * BitmapAnd, or BitmapOr node. + */ +static bool +plan_walk_members(List *plans, bool (*walker) (), void *context) +{ + ListCell *lc; + + foreach(lc, plans) + { + if (walker(lfirst(lc), context)) + return true; + } + + return false; +} diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 91a89b6d51..8364633d2e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -312,9 +312,11 @@ _outPlannedStmt(StringInfo str, const PlannedStmt *node) WRITE_BOOL_FIELD(transientPlan); WRITE_BOOL_FIELD(dependsOnRole); WRITE_BOOL_FIELD(parallelModeNeeded); + WRITE_BOOL_FIELD(usesPreExecPruning); WRITE_INT_FIELD(jitFlags); WRITE_NODE_FIELD(planTree); WRITE_NODE_FIELD(rtable); + WRITE_BITMAPSET_FIELD(relationRTIs); WRITE_NODE_FIELD(resultRelations); WRITE_NODE_FIELD(appendRelations); WRITE_NODE_FIELD(subplans); @@ -1003,6 +1005,8 @@ _outPartitionPruneInfo(StringInfo str, const PartitionPruneInfo *node) WRITE_NODE_TYPE("PARTITIONPRUNEINFO"); WRITE_NODE_FIELD(prune_infos); + WRITE_BOOL_FIELD(contains_init_steps); + WRITE_BOOL_FIELD(contains_exec_steps); WRITE_BITMAPSET_FIELD(other_subplans); } @@ -2273,6 +2277,7 @@ _outPlannerGlobal(StringInfo str, const PlannerGlobal *node) WRITE_NODE_FIELD(subplans); WRITE_BITMAPSET_FIELD(rewindPlanIDs); WRITE_NODE_FIELD(finalrtable); + WRITE_BITMAPSET_FIELD(relationRTIs); WRITE_NODE_FIELD(finalrowmarks); WRITE_NODE_FIELD(resultRelations); WRITE_NODE_FIELD(appendRelations); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index d79af6e56e..df06782c3c 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1585,9 +1585,11 @@ _readPlannedStmt(void) READ_BOOL_FIELD(transientPlan); READ_BOOL_FIELD(dependsOnRole); READ_BOOL_FIELD(parallelModeNeeded); + READ_BOOL_FIELD(usesPreExecPruning); READ_INT_FIELD(jitFlags); READ_NODE_FIELD(planTree); READ_NODE_FIELD(rtable); + READ_BITMAPSET_FIELD(relationRTIs); READ_NODE_FIELD(resultRelations); READ_NODE_FIELD(appendRelations); READ_NODE_FIELD(subplans); @@ -2533,6 +2535,8 @@ _readPartitionPruneInfo(void) READ_LOCALS(PartitionPruneInfo); READ_NODE_FIELD(prune_infos); + READ_BOOL_FIELD(contains_init_steps); + READ_BOOL_FIELD(contains_exec_steps); READ_BITMAPSET_FIELD(other_subplans); READ_DONE(); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index bd01ec0526..37a07cb258 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -517,8 +517,10 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, result->transientPlan = glob->transientPlan; result->dependsOnRole = glob->dependsOnRole; result->parallelModeNeeded = glob->parallelModeNeeded; + result->usesPreExecPruning = glob->usesPreExecPruning; result->planTree = top_plan; result->rtable = glob->finalrtable; + result->relationRTIs = glob->relationRTIs; result->resultRelations = glob->resultRelations; result->appendRelations = glob->appendRelations; result->subplans = glob->subplans; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 6ccec759bd..4616dc675d 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -483,6 +483,7 @@ static void add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte) { RangeTblEntry *newrte; + Index rti = list_length(glob->finalrtable) + 1; /* flat copy to duplicate all the scalar fields */ newrte = (RangeTblEntry *) palloc(sizeof(RangeTblEntry)); @@ -517,7 +518,10 @@ add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte) * but it would probably cost more cycles than it would save. */ if (newrte->rtekind == RTE_RELATION) + { + glob->relationRTIs = bms_add_member(glob->relationRTIs, rti); glob->relationOids = lappend_oid(glob->relationOids, newrte->relid); + } } /* @@ -1515,6 +1519,9 @@ set_append_references(PlannerInfo *root, pinfo->rtindex += rtoffset; } } + + if (aplan->part_prune_info->contains_init_steps) + root->glob->usesPreExecPruning = true; } /* We don't need to recurse to lefttree or righttree ... */ @@ -1579,6 +1586,9 @@ set_mergeappend_references(PlannerInfo *root, pinfo->rtindex += rtoffset; } } + + if (mplan->part_prune_info->contains_init_steps) + root->glob->usesPreExecPruning = true; } /* We don't need to recurse to lefttree or righttree ... */ diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index e00edbe5c8..d2874f716e 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -144,7 +144,9 @@ static List *make_partitionedrel_pruneinfo(PlannerInfo *root, List *prunequal, Bitmapset *partrelids, int *relid_subplan_map, - Bitmapset **matchedsubplans); + Bitmapset **matchedsubplans, + bool *contains_init_steps, + bool *contains_exec_steps); static void gen_partprune_steps(RelOptInfo *rel, List *clauses, PartClauseTarget target, GeneratePruningStepsContext *context); @@ -230,6 +232,8 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subplan_map; ListCell *lc; int i; + bool contains_init_steps = false; + bool contains_exec_steps = false; /* * Scan the subpaths to see which ones are scans of partition child @@ -309,12 +313,16 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, Bitmapset *partrelids = (Bitmapset *) lfirst(lc); List *pinfolist; Bitmapset *matchedsubplans = NULL; + bool partrel_contains_init_steps, + partrel_contains_exec_steps; pinfolist = make_partitionedrel_pruneinfo(root, parentrel, prunequal, partrelids, relid_subplan_map, - &matchedsubplans); + &matchedsubplans, + &partrel_contains_init_steps, + &partrel_contains_exec_steps); /* When pruning is possible, record the matched subplans */ if (pinfolist != NIL) @@ -323,6 +331,10 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, allmatchedsubplans = bms_join(matchedsubplans, allmatchedsubplans); } + if (!contains_init_steps) + contains_init_steps = partrel_contains_init_steps; + if (!contains_exec_steps) + contains_exec_steps = partrel_contains_exec_steps; } pfree(relid_subplan_map); @@ -337,6 +349,8 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, /* Else build the result data structure */ pruneinfo = makeNode(PartitionPruneInfo); pruneinfo->prune_infos = prunerelinfos; + pruneinfo->contains_init_steps = contains_init_steps; + pruneinfo->contains_exec_steps = contains_exec_steps; /* * Some subplans may not belong to any of the identified partitioned rels. @@ -435,13 +449,18 @@ add_part_relids(List *allpartrelids, Bitmapset *partrelids) * If we cannot find any useful run-time pruning steps, return NIL. * However, on success, each rel identified in partrelids will have * an element in the result list, even if some of them are useless. + * *contains_init_steps and *contains_exec_steps are set to indicate + * that the returned PartitionedRelPruneInfos contains pruning steps + * that can be performed before and during execution, respectively. */ static List * make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, List *prunequal, Bitmapset *partrelids, int *relid_subplan_map, - Bitmapset **matchedsubplans) + Bitmapset **matchedsubplans, + bool *contains_init_steps, + bool *contains_exec_steps) { RelOptInfo *targetpart = NULL; List *pinfolist = NIL; @@ -452,6 +471,10 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int rti; int i; + /* Will find out below. */ + *contains_init_steps = false; + *contains_exec_steps = false; + /* * Examine each partitioned rel, constructing a temporary array to map * from planner relids to index of the partitioned rel, and building a @@ -539,6 +562,9 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, * executor per-scan pruning steps. This first pass creates startup * pruning steps and detects whether there's any possibly-useful quals * that would require per-scan pruning. + * + * In the first pass, we note whether the 2nd pass is necessary by + * by noting the presence of EXEC parameters. */ gen_partprune_steps(subpart, partprunequal, PARTTARGET_INITIAL, &context); @@ -613,6 +639,11 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, pinfo->execparamids = execparamids; /* Remaining fields will be filled in the next loop */ + if (!*contains_init_steps) + *contains_init_steps = (initial_pruning_steps != NIL); + if (!*contains_exec_steps) + *contains_exec_steps = (exec_pruning_steps != NIL); + pinfolist = lappend(pinfolist, pinfo); } @@ -798,6 +829,7 @@ prune_append_rel_partitions(RelOptInfo *rel) /* These are not valid when being called from the planner */ context.planstate = NULL; + context.exprcontext = NULL; context.exprstates = NULL; /* Actual pruning happens here. */ @@ -808,8 +840,8 @@ prune_append_rel_partitions(RelOptInfo *rel) * get_matching_partitions * Determine partitions that survive partition pruning * - * Note: context->planstate must be set to a valid PlanState when the - * pruning_steps were generated with a target other than PARTTARGET_PLANNER. + * Note: context->exprcontext must be valid when the pruning_steps were + * generated with a target other than PARTTARGET_PLANNER. * * Returns a Bitmapset of the RelOptInfo->part_rels indexes of the surviving * partitions. @@ -3654,7 +3686,7 @@ match_boolean_partition_clause(Oid partopfamily, Expr *clause, Expr *partkey, * exprstate array. * * Note that the evaluated result may be in the per-tuple memory context of - * context->planstate->ps_ExprContext, and we may have leaked other memory + * context->exprcontext, and we may have leaked other memory * there too. This memory must be recovered by resetting that ExprContext * after we're done with the pruning operation (see execPartition.c). */ @@ -3677,13 +3709,18 @@ partkey_datum_from_expr(PartitionPruneContext *context, ExprContext *ectx; /* - * We should never see a non-Const in a step unless we're running in - * the executor. + * We should never see a non-Const in a step unless the caller has + * passed a valid ExprContext. + * + * When context->planstate is valid, context->exprcontext is same + * as context->planstate->ps_ExprContext. */ - Assert(context->planstate != NULL); + Assert(context->planstate != NULL || context->exprcontext != NULL); + Assert(context->planstate == NULL || + (context->exprcontext == context->planstate->ps_ExprContext)); exprstate = context->exprstates[stateidx]; - ectx = context->planstate->ps_ExprContext; + ectx = context->exprcontext; *value = ExecEvalExprSwitchContext(exprstate, ectx, isnull); } } diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index 6767eae8f2..6161907ace 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -58,6 +58,7 @@ #include "access/transam.h" #include "catalog/namespace.h" +#include "executor/execPartition.h" #include "executor/executor.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" @@ -99,14 +100,26 @@ static dlist_head cached_expression_list = DLIST_STATIC_INIT(cached_expression_l static void ReleaseGenericPlan(CachedPlanSource *plansource); static List *RevalidateCachedQuery(CachedPlanSource *plansource, QueryEnvironment *queryEnv); -static bool CheckCachedPlan(CachedPlanSource *plansource); +static bool CheckCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams); static CachedPlan *BuildCachedPlan(CachedPlanSource *plansource, List *qlist, ParamListInfo boundParams, QueryEnvironment *queryEnv); static bool choose_custom_plan(CachedPlanSource *plansource, ParamListInfo boundParams); static double cached_plan_cost(CachedPlan *plan, bool include_planner); static Query *QueryListGetPrimaryStmt(List *stmts); -static void AcquireExecutorLocks(List *stmt_list, bool acquire); +static void AcquireExecutorLocks(List *stmt_list, bool acquire, + ParamListInfo boundParams); +struct GetLockableRelations_context +{ + PlannedStmt *plannedstmt; + Bitmapset *relations; + ParamListInfo params; +}; +static Bitmapset *GetLockableRelations(PlannedStmt *plannedstmt, + ParamListInfo boundParams); +static bool GetLockableRelations_worker(Plan *plan, + struct GetLockableRelations_context *context); +static Bitmapset *get_plan_scanrelids(Plan *plan); static void AcquirePlannerLocks(List *stmt_list, bool acquire); static void ScanQueryForLocks(Query *parsetree, bool acquire); static bool ScanQueryWalker(Node *node, bool *acquire); @@ -792,7 +805,7 @@ RevalidateCachedQuery(CachedPlanSource *plansource, * (We must do this for the "true" result to be race-condition-free.) */ static bool -CheckCachedPlan(CachedPlanSource *plansource) +CheckCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams) { CachedPlan *plan = plansource->gplan; @@ -826,7 +839,7 @@ CheckCachedPlan(CachedPlanSource *plansource) */ Assert(plan->refcount > 0); - AcquireExecutorLocks(plan->stmt_list, true); + AcquireExecutorLocks(plan->stmt_list, true, boundParams); /* * If plan was transient, check to see if TransactionXmin has @@ -848,7 +861,7 @@ CheckCachedPlan(CachedPlanSource *plansource) } /* Oops, the race case happened. Release useless locks. */ - AcquireExecutorLocks(plan->stmt_list, false); + AcquireExecutorLocks(plan->stmt_list, false, boundParams); } /* @@ -1160,7 +1173,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, if (!customplan) { - if (CheckCachedPlan(plansource)) + if (CheckCachedPlan(plansource, boundParams)) { /* We want a generic plan, and we already have a valid one */ plan = plansource->gplan; @@ -1366,7 +1379,6 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, foreach(lc, plan->stmt_list) { PlannedStmt *plannedstmt = lfirst_node(PlannedStmt, lc); - ListCell *lc2; if (plannedstmt->commandType == CMD_UTILITY) return false; @@ -1375,13 +1387,8 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, * We have to grovel through the rtable because it's likely to contain * an RTE_RESULT relation, rather than being totally empty. */ - foreach(lc2, plannedstmt->rtable) - { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc2); - - if (rte->rtekind == RTE_RELATION) - return false; - } + if (!bms_is_empty(plannedstmt->relationRTIs)) + return false; } /* @@ -1740,14 +1747,15 @@ QueryListGetPrimaryStmt(List *stmts) * or release them if acquire is false. */ static void -AcquireExecutorLocks(List *stmt_list, bool acquire) +AcquireExecutorLocks(List *stmt_list, bool acquire, ParamListInfo boundParams) { ListCell *lc1; foreach(lc1, stmt_list) { PlannedStmt *plannedstmt = lfirst_node(PlannedStmt, lc1); - ListCell *lc2; + Bitmapset *relations; + int rti; if (plannedstmt->commandType == CMD_UTILITY) { @@ -1765,9 +1773,22 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) continue; } - foreach(lc2, plannedstmt->rtable) + /* + * Fetch the RT indexes of only the relations that will be actually + * scanned when the plan is executed. This skips over scan nodes + * appearing as child subnodes of any Append/MergeAppend nodes present + * in the plan tree. It does so by performing + * ExecFindInitialMatchingSubPlans() to run any pruning steps + * contained in those nodes that can be safely run at this point, using + * 'boundParams' to evaluate any EXTERN parameters contained in the + * steps. + */ + relations = GetLockableRelations(plannedstmt, boundParams); + + rti = -1; + while ((rti = bms_next_member(relations, rti)) >= 0) { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc2); + RangeTblEntry *rte = rt_fetch(rti, plannedstmt->rtable); if (rte->rtekind != RTE_RELATION) continue; @@ -1786,6 +1807,166 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) } } +/* + * GetLockableRelations + * Returns set of RT indexes of relations that must be locked by + * AcquireExecutorLocks() + */ +static Bitmapset * +GetLockableRelations(PlannedStmt *plannedstmt, ParamListInfo boundParams) +{ + ListCell *lc; + struct GetLockableRelations_context context; + + /* None of the relation scanning nodes are prunable here. */ + if (!plannedstmt->usesPreExecPruning) + return plannedstmt->relationRTIs; + + /* + * Look for prunable nodes in the main plan tree, followed by those in + * subplans. + */ + context.plannedstmt = plannedstmt; + context.params = boundParams; + context.relations = NULL; + + (void) GetLockableRelations_worker(plannedstmt->planTree, &context); + + foreach(lc, plannedstmt->subplans) + { + Plan *subplan = lfirst(lc); + + (void) GetLockableRelations_worker(subplan, &context); + } + + return context.relations; +} + +/* + * GetLockableRelations_worker + * Adds RT indexes of relations to be scanned by plan to + * context->relations + * + * For plan node types that support pruning, this only adds child plan + * subnodes that satisfy the "initial" pruning steps. + */ +static bool +GetLockableRelations_worker(Plan *plan, + struct GetLockableRelations_context *context) +{ + if (plan == NULL) + return false; + + switch(nodeTag(plan)) + { + /* Nodes scanning a relation or relations. */ + case T_SeqScan: + case T_SampleScan: + case T_IndexScan: + case T_IndexOnlyScan: + case T_BitmapHeapScan: + case T_TidScan: + case T_TidRangeScan: + context->relations = bms_add_member(context->relations, + ((Scan *) plan)->scanrelid); + return false; + case T_ForeignScan: + context->relations = bms_add_members(context->relations, + ((ForeignScan *) plan)->fs_relids); + return false; + case T_CustomScan: + context->relations = bms_add_members(context->relations, + ((CustomScan *) plan)->custom_relids); + return false; + + /* Nodes containing prunable subnodes. */ + case T_Append: + case T_MergeAppend: + { + PlannedStmt *plannedstmt = context->plannedstmt; + List *rtable = plannedstmt->rtable; + ParamListInfo params = context->params; + PartitionPruneInfo *pruneinfo; + Bitmapset *validsubplans; + Bitmapset *parentrelids; + + pruneinfo = IsA(plan, Append) ? + ((Append *) plan)->part_prune_info : + ((MergeAppend *) plan)->part_prune_info; + + if (pruneinfo && pruneinfo->contains_init_steps) + { + int i; + List *subplans = IsA(plan, Append) ? + ((Append *) plan)->appendplans : + ((MergeAppend *) plan)->mergeplans; + + validsubplans = + ExecFindInitialMatchingSubPlans(pruneinfo, + NULL, rtable, + params, + &parentrelids); + + /* All relevant parents must be locked. */ + Assert(bms_num_members(parentrelids) > 0); + context->relations = bms_add_members(context->relations, + parentrelids); + + /* And all leaf partitions that will be scanned. */ + i = -1; + while ((i = bms_next_member(validsubplans, i)) >= 0) + { + Plan *subplan = list_nth(subplans, i); + + context->relations = + bms_add_members(context->relations, + get_plan_scanrelids(subplan)); + } + + return false; + } + } + break; + + default: + break; + } + + return plan_tree_walker(plan, GetLockableRelations_worker, + (void *) context); +} + +/* + * get_plan_scanrelid + * Returns RT indexes of the relation(s) scanned by plan + */ +static Bitmapset * +get_plan_scanrelids(Plan *plan) +{ + if (plan == NULL) + return NULL; + + switch(nodeTag(plan)) + { + case T_SeqScan: + case T_SampleScan: + case T_IndexScan: + case T_IndexOnlyScan: + case T_BitmapHeapScan: + case T_TidScan: + case T_TidRangeScan: + return bms_make_singleton(((Scan *) plan)->scanrelid); + case T_ForeignScan: + return ((ForeignScan *) plan)->fs_relids; + case T_CustomScan: + return ((CustomScan *) plan)->custom_relids; + default: + break; + } + + return NULL; +} + /* * AcquirePlannerLocks: acquire locks needed for planning of a querytree list; * or release them if acquire is false. diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b7dd..0eeaf3e79d 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -90,8 +90,6 @@ typedef struct PartitionPruningData * These must not be pruned. * prune_context A short-lived memory context in which to execute the * partition pruning functions. - * do_initial_prune true if pruning should be performed during executor - * startup (at any hierarchy level). * do_exec_prune true if pruning should be performed during * executor run (at any hierarchy level). * num_partprunedata Number of items in "partprunedata" array. @@ -104,7 +102,6 @@ typedef struct PartitionPruneState Bitmapset *execparamids; Bitmapset *other_subplans; MemoryContext prune_context; - bool do_initial_prune; bool do_exec_prune; int num_partprunedata; PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; @@ -120,9 +117,13 @@ extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, extern void ExecCleanupTupleRouting(ModifyTableState *mtstate, PartitionTupleRouting *proute); extern PartitionPruneState *ExecCreatePartitionPruneState(PlanState *planstate, - PartitionPruneInfo *partitionpruneinfo); + PartitionPruneInfo *partitionpruneinfo, + Bitmapset *initially_valid_subplans, + int nsubplans); extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruneState *prunestate); -extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, - int nsubplans); +extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruneInfo *pruneinfo, + EState *estate, List *rtable, + ParamListInfo params, + Bitmapset **parentrelids); #endif /* EXECPARTITION_H */ diff --git a/src/include/nodes/nodeFuncs.h b/src/include/nodes/nodeFuncs.h index 03a346c01d..8b985a4706 100644 --- a/src/include/nodes/nodeFuncs.h +++ b/src/include/nodes/nodeFuncs.h @@ -158,5 +158,8 @@ extern bool raw_expression_tree_walker(Node *node, bool (*walker) (), struct PlanState; extern bool planstate_tree_walker(struct PlanState *planstate, bool (*walker) (), void *context); +struct Plan; +extern bool plan_tree_walker(struct Plan *plan, bool (*walker) (), + void *context); #endif /* NODEFUNCS_H */ diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 324d92880b..d041b4d924 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -101,6 +101,9 @@ typedef struct PlannerGlobal List *finalrtable; /* "flat" rangetable for executor */ + Bitmapset *relationRTIs; /* Indexes of RTE_RELATION entries in range + * table */ + List *finalrowmarks; /* "flat" list of PlanRowMarks */ List *resultRelations; /* "flat" list of integer RT indexes */ @@ -129,6 +132,9 @@ typedef struct PlannerGlobal char maxParallelHazard; /* worst PROPARALLEL hazard level */ + bool usesPreExecPruning; /* Do some Plan nodes use pre-execution + * partition pruning */ + PartitionDirectory partition_directory; /* partition descriptors */ } PlannerGlobal; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index be3c30704a..23bf04578b 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -59,12 +59,18 @@ typedef struct PlannedStmt bool parallelModeNeeded; /* parallel mode required to execute? */ + bool usesPreExecPruning; /* Do some Plan nodes use pre-execution + * partition pruning */ + int jitFlags; /* which forms of JIT should be performed */ struct Plan *planTree; /* tree of Plan nodes */ List *rtable; /* list of RangeTblEntry nodes */ + Bitmapset *relationRTIs; /* Indexes of RTE_RELATION entries in range + * table */ + /* rtable indexes of target relations for INSERT/UPDATE/DELETE */ List *resultRelations; /* integer list of RT indexes, or NIL */ @@ -1157,6 +1163,13 @@ typedef struct PlanRowMark * prune_infos List of Lists containing PartitionedRelPruneInfo nodes, * one sublist per run-time-prunable partition hierarchy * appearing in the parent plan node's subplans. + * + * contains_init_steps Does any of the PartitionedRelPruneInfos in + * prune_infos have its initial_pruning_steps set? + * + * contains_exec_steps Does any of the PartitionedRelPruneInfos in + * prune_infos have its exec_pruning_steps set? + * * other_subplans Indexes of any subplans that are not accounted for * by any of the PartitionedRelPruneInfo nodes in * "prune_infos". These subplans must not be pruned. @@ -1165,6 +1178,8 @@ typedef struct PartitionPruneInfo { NodeTag type; List *prune_infos; + bool contains_init_steps; + bool contains_exec_steps; Bitmapset *other_subplans; } PartitionPruneInfo; diff --git a/src/include/partitioning/partprune.h b/src/include/partitioning/partprune.h index 5f51e73a4d..1c9c408f00 100644 --- a/src/include/partitioning/partprune.h +++ b/src/include/partitioning/partprune.h @@ -41,6 +41,8 @@ struct RelOptInfo; * subsidiary data, such as the FmgrInfos. * planstate Points to the parent plan node's PlanState when called * during execution; NULL when called from the planner. + * exprcontext ExprContext to use during pre-execution pruning; planstate + * would be NULL in that case. * exprstates Array of ExprStates, indexed as per PruneCxtStateIdx; one * for each partition key in each pruning step. Allocated if * planstate is non-NULL, otherwise NULL. @@ -56,6 +58,7 @@ typedef struct PartitionPruneContext FmgrInfo *stepcmpfuncs; MemoryContext ppccontext; PlanState *planstate; + ExprContext *exprcontext; ExprState **exprstates; } PartitionPruneContext; -- 2.24.1