From 5e78a80ad51b1e9436c43d3422056cdc8d1117d3 Mon Sep 17 00:00:00 2001 From: Tatsuro Yamada Date: Wed, 26 Jun 2024 17:03:34 +0900 Subject: [PATCH] Add Showing applied extended statistics in explain r3. This patch fixes the following points of issues as Tomas mentioned before: 1. Previously, clauses in Grouping queries were incorrectly displayed as And lists, but have now been fixed to display them as comma lists. 2. To reduce overhead, extended statistics information is now tracked only when the verbose option is selected. Also, The strings displayed in Explain were replaced with the following: s/Statistics/Ext Stats/ --- src/backend/commands/explain.c | 114 ++++++++++++++++++++++ src/backend/nodes/makefuncs.c | 11 +++ src/backend/optimizer/plan/createplan.c | 15 +++ src/backend/optimizer/util/relnode.c | 12 +++ src/backend/optimizer/util/restrictinfo.c | 35 +++++++ src/backend/statistics/extended_stats.c | 8 ++ src/backend/utils/adt/selfuncs.c | 15 +++ src/backend/utils/cache/lsyscache.c | 49 ++++++++++ src/include/nodes/makefuncs.h | 2 + src/include/nodes/parsenodes.h | 3 + src/include/nodes/pathnodes.h | 5 + src/include/nodes/plannodes.h | 5 + src/include/optimizer/restrictinfo.h | 2 + src/include/utils/lsyscache.h | 3 + 14 files changed, 279 insertions(+) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 94511a5a02..7e39d37b59 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -94,6 +94,9 @@ static void show_qual(List *qual, const char *qlabel, static void show_scan_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, ExplainState *es); +static void show_scan_stats(List *stats, List *clauses, List *ors, + PlanState *planstate, List *ancestors, + ExplainState *es); static void show_upper_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, ExplainState *es); @@ -483,6 +486,11 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, if (es->buffers) bufusage_start = pgBufferUsage; + + /* if this flag is true, applied ext stats are stored */ + if (es->verbose) + query->isExplain = true; + INSTR_TIME_SET_CURRENT(planstart); /* plan the query */ @@ -1975,6 +1983,10 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->verbose) + show_scan_stats(plan->applied_stats, plan->applied_clauses, + plan->applied_clauses_or, + planstate, ancestors, es); break; case T_IndexOnlyScan: show_scan_qual(((IndexOnlyScan *) plan)->indexqual, @@ -1991,10 +2003,18 @@ ExplainNode(PlanState *planstate, List *ancestors, if (es->analyze) ExplainPropertyFloat("Heap Fetches", NULL, planstate->instrument->ntuples2, 0, es); + if (es->verbose) + show_scan_stats(plan->applied_stats, plan->applied_clauses, + plan->applied_clauses_or, + planstate, ancestors, es); break; case T_BitmapIndexScan: show_scan_qual(((BitmapIndexScan *) plan)->indexqualorig, "Index Cond", planstate, ancestors, es); + if (es->verbose) + show_scan_stats(plan->applied_stats, plan->applied_clauses, + plan->applied_clauses_or, + planstate, ancestors, es); break; case T_BitmapHeapScan: show_scan_qual(((BitmapHeapScan *) plan)->bitmapqualorig, @@ -2024,6 +2044,10 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->verbose) + show_scan_stats(plan->applied_stats, plan->applied_clauses, + plan->applied_clauses_or, + planstate, ancestors, es); break; case T_Gather: { @@ -2203,6 +2227,10 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->verbose) + show_scan_stats(plan->applied_stats, plan->applied_clauses, + plan->applied_clauses_or, + planstate, ancestors, es); break; case T_WindowAgg: show_upper_qual(plan->qual, "Filter", planstate, ancestors, es); @@ -2218,6 +2246,10 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->verbose) + show_scan_stats(plan->applied_stats, plan->applied_clauses, + plan->applied_clauses_or, + planstate, ancestors, es); break; case T_Sort: show_sort_keys(castNode(SortState, planstate), ancestors, es); @@ -2537,6 +2569,88 @@ show_scan_qual(List *qual, const char *qlabel, show_qual(qual, qlabel, planstate, ancestors, useprefix, es); } +/* + * Show a generic expression + */ +static char * +deparse_stat_expression(Node *node, + PlanState *planstate, List *ancestors, + ExplainState *es) +{ + List *context; + + /* Set up deparsing context */ + context = set_deparse_context_plan(es->deparse_cxt, + planstate->plan, + ancestors); + + /* Deparse the expression */ + return deparse_expression(node, context, false, false); +} + +/* + * Show a qualifier expression (which is a List with implicit AND semantics) + */ +static char * +show_stat_qual(List *qual, int is_or, + PlanState *planstate, List *ancestors, + ExplainState *es) +{ + Node *node; + + /* No work if empty qual */ + if (qual == NIL) + return NULL; + + /* Convert AND list to explicit AND */ + switch (is_or) + { + case 0: + node = (Node *) make_ands_explicit(qual); + break; + case 1: + node = (Node *) make_ors_explicit(qual); + break; + case 2: + /* Extended stats for GROUP BY clause should be comma separeted string */ + node = (Node *) qual; + break; + default: + elog(ERROR, "unexpected value: %d", is_or); + break; + } + + /* And show it */ + return deparse_stat_expression(node, planstate, ancestors, es); +} + +/* + * Show applied statistics for scan/agg/group plan node + */ +static void +show_scan_stats(List *stats, List *clauses, List *ors, + PlanState *planstate, List *ancestors, ExplainState *es) +{ + ListCell *lc1, *lc2, *lc3; + StringInfoData str; + + forthree (lc1, stats, lc2, clauses, lc3, ors) + { + StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(lc1); + List *applied_clauses = (List *) lfirst(lc2); + int is_or = lfirst_int(lc3); + + initStringInfo(&str); + + appendStringInfo(&str, "%s.%s Clauses: %s", + get_namespace_name(get_statistics_namespace(stat->statOid)), + get_statistics_name(stat->statOid), + show_stat_qual(applied_clauses, is_or, planstate, ancestors, es)); + + ExplainPropertyText("Ext Stats", str.data, es); + } +} + /* * Show a qualifier expression for an upper-level plan node */ diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c index 61ac172a85..725c76ab63 100644 --- a/src/backend/nodes/makefuncs.c +++ b/src/backend/nodes/makefuncs.c @@ -733,6 +733,17 @@ make_ands_explicit(List *andclauses) return make_andclause(andclauses); } +Expr * +make_ors_explicit(List *orclauses) +{ + if (orclauses == NIL) + return (Expr *) makeBoolConst(true, false); + else if (list_length(orclauses) == 1) + return (Expr *) linitial(orclauses); + else + return make_orclause(orclauses); +} + List * make_ands_implicit(Expr *clause) { diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 6b64c4a362..25541308f4 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -5408,12 +5408,27 @@ order_qual_clauses(PlannerInfo *root, List *clauses) static void copy_generic_path_info(Plan *dest, Path *src) { + ListCell *lc; + dest->startup_cost = src->startup_cost; dest->total_cost = src->total_cost; dest->plan_rows = src->rows; dest->plan_width = src->pathtarget->width; dest->parallel_aware = src->parallel_aware; dest->parallel_safe = src->parallel_safe; + + dest->applied_stats = src->parent->applied_stats; + dest->applied_clauses_or = src->parent->applied_clauses_or; + + dest->applied_clauses = NIL; + foreach (lc, src->parent->applied_clauses) + { + List *clauses = (List *) lfirst(lc); + + dest->applied_clauses + = lappend(dest->applied_clauses, + maybe_extract_actual_clauses(clauses, false)); + } } /* diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index e05b21c884..637f730a8e 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -287,6 +287,10 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->partexprs = NULL; rel->nullable_partexprs = NULL; + rel->applied_stats = NIL; + rel->applied_clauses = NIL; + rel->applied_clauses_or = NIL; + /* * Pass assorted information down the inheritance hierarchy. */ @@ -769,6 +773,10 @@ build_join_rel(PlannerInfo *root, joinrel->partexprs = NULL; joinrel->nullable_partexprs = NULL; + joinrel->applied_stats = NIL; + joinrel->applied_clauses = NIL; + joinrel->applied_clauses_or = NIL; + /* Compute information relevant to the foreign relations. */ set_foreign_rel_properties(joinrel, outer_rel, inner_rel); @@ -963,6 +971,10 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->partexprs = NULL; joinrel->nullable_partexprs = NULL; + joinrel->applied_stats = NIL; + joinrel->applied_clauses = NIL; + joinrel->applied_clauses_or = NIL; + /* Compute information relevant to foreign relations. */ set_foreign_rel_properties(joinrel, outer_rel, inner_rel); diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 0b406e9334..23136a330b 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -508,6 +508,41 @@ extract_actual_clauses(List *restrictinfo_list, return result; } +/* + * maybe_extract_actual_clauses + * + * Just like extract_actual_clauses, but does not require the clauses to + * already be RestrictInfo. + * + * XXX Does not handle RestrictInfos nested in OR clauses. + */ +List * +maybe_extract_actual_clauses(List *restrictinfo_list, + bool pseudoconstant) +{ + List *result = NIL; + ListCell *l; + + foreach(l, restrictinfo_list) + { + RestrictInfo *rinfo; + Node *node = (Node *) lfirst(l); + + if (!IsA(node, RestrictInfo)) + { + result = lappend(result, node); + continue; + } + + rinfo = (RestrictInfo *) node; + + if (rinfo->pseudoconstant == pseudoconstant) + result = lappend(result, rinfo->clause); + } + + return result; +} + /* * extract_actual_join_clauses * diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 99fdf208db..4d92f24c61 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -1857,6 +1857,14 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli list_exprs[listidx] = NULL; } + /* add it to the list of applied stats/clauses, if this flag is true */ + if (root->parse->isExplain) + { + rel->applied_stats = lappend(rel->applied_stats, stat); + rel->applied_clauses = lappend(rel->applied_clauses, stat_clauses); + rel->applied_clauses_or = lappend_int(rel->applied_clauses_or, (is_or) ? 1 : 0); + } + if (is_or) { bool *or_matches = NULL; diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 5f5d7959d8..2aec61e7bb 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -4069,6 +4069,7 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, ListCell *lc2; Bitmapset *matched = NULL; AttrNumber attnum_offset; + List *matched_exprs = NIL; /* * How much we need to offset the attnums? If there are no @@ -4116,6 +4117,9 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, matched = bms_add_member(matched, attnum); + /* track expressions matched by this statistics */ + matched_exprs = lappend(matched_exprs, varinfo->var); + found = true; } @@ -4144,6 +4148,9 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, matched = bms_add_member(matched, attnum); + /* track expressions matched by this statistics */ + matched_exprs = lappend(matched_exprs, expr); + /* there should be just one matching expression */ break; } @@ -4152,6 +4159,14 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, } } + /* add it to the list of applied stats/clauses, if this flag is true */ + if (root->parse->isExplain) + { + rel->applied_stats = lappend(rel->applied_stats, matched_info); + rel->applied_clauses = lappend(rel->applied_clauses, matched_exprs); + rel->applied_clauses_or = lappend_int(rel->applied_clauses_or, 2); /* 2: Use comma to deparse */ + } + /* Find the specific item that exactly matches the combination */ for (i = 0; i < stats->nitems; i++) { diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 48a280d089..d135751dd0 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -35,6 +35,7 @@ #include "catalog/pg_publication.h" #include "catalog/pg_range.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_transform.h" #include "catalog/pg_type.h" @@ -3714,3 +3715,51 @@ get_subscription_name(Oid subid, bool missing_ok) return subname; } + +/* + * get_statistics_name + * Returns the name of a given extended statistics + * + * Returns a palloc'd copy of the string, or NULL if no such namespace. + */ +char * +get_statistics_name(Oid stxid) +{ + HeapTuple tp; + + tp = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(stxid)); + if (HeapTupleIsValid(tp)) + { + Form_pg_statistic_ext stxtup = (Form_pg_statistic_ext) GETSTRUCT(tp); + char *result; + + result = pstrdup(NameStr(stxtup->stxname)); + ReleaseSysCache(tp); + return result; + } + else + return NULL; +} + +/* + * get_statistics_namespace + * Returns the namespace OID of a given extended statistics + */ +Oid +get_statistics_namespace(Oid stxid) +{ + HeapTuple tp; + + tp = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(stxid)); + if (HeapTupleIsValid(tp)) + { + Form_pg_statistic_ext stxtup = (Form_pg_statistic_ext) GETSTRUCT(tp); + Oid result; + + result = stxtup->stxnamespace; + ReleaseSysCache(tp); + return result; + } + else + return InvalidOid; +} diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h index 5209d3de89..f5ae402c16 100644 --- a/src/include/nodes/makefuncs.h +++ b/src/include/nodes/makefuncs.h @@ -94,6 +94,8 @@ extern Node *make_and_qual(Node *qual1, Node *qual2); extern Expr *make_ands_explicit(List *andclauses); extern List *make_ands_implicit(Expr *clause); +extern Expr *make_ors_explicit(List *orclauses); + extern IndexInfo *makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid, List *expressions, List *predicates, bool unique, bool nulls_not_distinct, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 85a62b538e..36b479c104 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -238,6 +238,9 @@ typedef struct Query ParseLoc stmt_location; /* length in bytes; 0 means "rest of string" */ ParseLoc stmt_len pg_node_attr(query_jumble_ignore); + + /* if true, query is explain verbose */ + bool isExplain; } Query; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 2ba297c117..d78f08c3c4 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1043,6 +1043,11 @@ typedef struct RelOptInfo List **partexprs pg_node_attr(read_write_ignore); /* Nullable partition key expressions */ List **nullable_partexprs pg_node_attr(read_write_ignore); + + /* info about applied extended statistics */ + List *applied_stats; /* list of StatisticExtInfo */ + List *applied_clauses; /* list of lists of clauses */ + List *applied_clauses_or; /* are the clauses AND, OR, or Comma */ } RelOptInfo; /* diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 1aeeaec95e..504e5cc475 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -169,6 +169,11 @@ typedef struct Plan */ Bitmapset *extParam; Bitmapset *allParam; + + /* info about applied statistics */ + List *applied_stats; + List *applied_clauses; + List *applied_clauses_or; } Plan; /* ---------------- diff --git a/src/include/optimizer/restrictinfo.h b/src/include/optimizer/restrictinfo.h index 1b42c832c5..29aa519b99 100644 --- a/src/include/optimizer/restrictinfo.h +++ b/src/include/optimizer/restrictinfo.h @@ -39,6 +39,8 @@ extern bool restriction_is_securely_promotable(RestrictInfo *restrictinfo, extern List *get_actual_clauses(List *restrictinfo_list); extern List *extract_actual_clauses(List *restrictinfo_list, bool pseudoconstant); +extern List *maybe_extract_actual_clauses(List *restrictinfo_list, + bool pseudoconstant); extern void extract_actual_join_clauses(List *restrictinfo_list, Relids joinrelids, List **joinquals, diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 20446f6f83..3fab6d5eea 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -206,6 +206,9 @@ extern char *get_publication_name(Oid pubid, bool missing_ok); extern Oid get_subscription_oid(const char *subname, bool missing_ok); extern char *get_subscription_name(Oid subid, bool missing_ok); +extern char *get_statistics_name(Oid stxid); +extern Oid get_statistics_namespace(Oid stxid); + #define type_is_array(typid) (get_element_type(typid) != InvalidOid) /* type_is_array_domain accepts both plain arrays and domains over arrays */ #define type_is_array_domain(typid) (get_base_element_type(typid) != InvalidOid) -- 2.39.3