From 0bc86ef1e0de8b0f1b9c511eef56c007b0da1153 Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Fri, 25 Nov 2022 11:23:20 -0800 Subject: [PATCH v4 1/2] Add autovacuum trigger instrumentation. Add new instrumentation that lists a triggering condition in the server log whenever an autovacuum is logged. This reports "table age" as the triggering criteria when antiwraparound autovacuum runs (the XID age trigger case and the MXID age trigger case are represented separately). Other cases are reported as autovacuum trigger when the tuple insert thresholds or the dead tuple thresholds were crossed. Author: Peter Geoghegan Reviewed-By: Jeff Davis Discussion: https://postgr.es/m/CAH2-Wz=S-R_2rO49Hm94Nuvhu9_twRGbTm6uwDRmRu-Sqn_t3w@mail.gmail.com --- src/include/commands/vacuum.h | 18 ++- src/backend/access/heap/vacuumlazy.c | 14 +++ src/backend/commands/vacuum.c | 15 ++- src/backend/postmaster/autovacuum.c | 165 +++++++++++++++++---------- 4 files changed, 145 insertions(+), 67 deletions(-) diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 689dbb770..b70f69fd9 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -191,6 +191,21 @@ typedef struct VacAttrStats #define VACOPT_SKIP_DATABASE_STATS 0x100 /* skip vac_update_datfrozenxid() */ #define VACOPT_ONLY_DATABASE_STATS 0x200 /* only vac_update_datfrozenxid() */ +/* + * Values used by autovacuum.c to tell vacuumlazy.c about the specific + * threshold type that triggered an autovacuum worker. + * + * AUTOVACUUM_NONE is used when VACUUM isn't running in an autovacuum worker. + */ +typedef enum AutoVacType +{ + AUTOVACUUM_NONE = 0, + AUTOVACUUM_TABLE_XID_AGE, + AUTOVACUUM_TABLE_MXID_AGE, + AUTOVACUUM_DEAD_TUPLES, + AUTOVACUUM_INSERTED_TUPLES, +} AutoVacType; + /* * Values used by index_cleanup and truncate params. * @@ -222,7 +237,8 @@ typedef struct VacuumParams * use default */ int multixact_freeze_table_age; /* multixact age at which to scan * whole table */ - bool is_wraparound; /* force a for-wraparound vacuum */ + bool is_wraparound; /* antiwraparound autovacuum? */ + AutoVacType trigger; /* Autovacuum trigger condition, if any */ int log_min_duration; /* minimum execution threshold in ms at * which autovacuum is logged, -1 to use * default */ diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 369451516..083901922 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -639,6 +639,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * implies aggressive. Produce distinct output for the corner * case all the same, just in case. */ + Assert(params->trigger == AUTOVACUUM_TABLE_XID_AGE || + params->trigger == AUTOVACUUM_TABLE_MXID_AGE); if (vacrel->aggressive) msgfmt = _("automatic aggressive vacuum to prevent wraparound of table \"%s.%s.%s\": index scans: %d\n"); else @@ -656,6 +658,18 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->relnamespace, vacrel->relname, vacrel->num_index_scans); + if (!verbose) + { + Assert(params->trigger != AUTOVACUUM_NONE); + if (params->trigger == AUTOVACUUM_TABLE_XID_AGE) + appendStringInfo(&buf, _("autovacuum trigger: table XID age threshold\n")); + else if (params->trigger == AUTOVACUUM_TABLE_MXID_AGE) + appendStringInfo(&buf, _("autovacuum trigger: table MultiXactId age threshold\n")); + else if (params->trigger == AUTOVACUUM_DEAD_TUPLES) + appendStringInfo(&buf, _("autovacuum trigger: dead tuples threshold\n")); + else if (params->trigger == AUTOVACUUM_INSERTED_TUPLES) + appendStringInfo(&buf, _("autovacuum trigger: inserted tuples threshold\n")); + } appendStringInfo(&buf, _("pages: %u removed, %u remain, %u scanned (%.2f%% of total)\n"), vacrel->removed_pages, new_rel_pages, diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index c4ed7efce..d17fbab17 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -271,8 +271,9 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) params.multixact_freeze_table_age = -1; } - /* user-invoked vacuum is never "for wraparound" */ + /* user-invoked vacuum never uses these autovacuum-only flags */ params.is_wraparound = false; + params.trigger = AUTOVACUUM_NONE; /* user-invoked vacuum uses VACOPT_VERBOSE instead of log_min_duration */ params.log_min_duration = -1; @@ -1049,8 +1050,8 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, /* * Determine the minimum freeze age to use: as specified by the caller, or * vacuum_freeze_min_age, but in any case not more than half - * autovacuum_freeze_max_age, so that autovacuums to prevent XID - * wraparound won't occur too frequently. + * autovacuum_freeze_max_age, so that table XID age autovacuums won't + * occur too frequently. */ if (freeze_min_age < 0) freeze_min_age = vacuum_freeze_min_age; @@ -1068,8 +1069,8 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, /* * Determine the minimum multixact freeze age to use: as specified by * caller, or vacuum_multixact_freeze_min_age, but in any case not more - * than half effective_multixact_freeze_max_age, so that autovacuums to - * prevent MultiXact wraparound won't occur too frequently. + * than half effective_multixact_freeze_max_age, so that table MXID age + * autovacuums won't occur too frequently. */ if (multixact_freeze_min_age < 0) multixact_freeze_min_age = vacuum_multixact_freeze_min_age; @@ -1870,7 +1871,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params) LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); MyProc->statusFlags |= PROC_IN_VACUUM; if (params->is_wraparound) + { + Assert(params->trigger == AUTOVACUUM_TABLE_XID_AGE || + params->trigger == AUTOVACUUM_TABLE_MXID_AGE); MyProc->statusFlags |= PROC_VACUUM_FOR_WRAPAROUND; + } ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags; LWLockRelease(ProcArrayLock); } diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index f5ea381c5..d428ef066 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -327,15 +327,17 @@ static void FreeWorkerInfo(int code, Datum arg); static autovac_table *table_recheck_autovac(Oid relid, HTAB *table_toast_map, TupleDesc pg_class_desc, int effective_multixact_freeze_max_age); -static void recheck_relation_needs_vacanalyze(Oid relid, AutoVacOpts *avopts, - Form_pg_class classForm, - int effective_multixact_freeze_max_age, - bool *dovacuum, bool *doanalyze, bool *wraparound); -static void relation_needs_vacanalyze(Oid relid, AutoVacOpts *relopts, - Form_pg_class classForm, - PgStat_StatTabEntry *tabentry, - int effective_multixact_freeze_max_age, - bool *dovacuum, bool *doanalyze, bool *wraparound); +static AutoVacType recheck_relation_needs_vacanalyze(Oid relid, AutoVacOpts *avopts, + Form_pg_class classForm, + int effective_multixact_freeze_max_age, + bool *dovacuum, bool *doanalyze, + bool *wraparound); +static AutoVacType relation_needs_vacanalyze(Oid relid, AutoVacOpts *relopts, + Form_pg_class classForm, + PgStat_StatTabEntry *tabentry, + int effective_multixact_freeze_max_age, + bool *dovacuum, bool *doanalyze, + bool *wraparound); static void autovacuum_do_vac_analyze(autovac_table *tab, BufferAccessStrategy bstrategy); @@ -1148,8 +1150,8 @@ do_start_worker(void) { List *dblist; ListCell *cell; - TransactionId xidForceLimit; - MultiXactId multiForceLimit; + TransactionId xidAgeLimit; + MultiXactId multiAgeLimit; bool for_xid_wrap; bool for_multi_wrap; avw_dbase *avdb; @@ -1186,17 +1188,17 @@ do_start_worker(void) * particular tables, but not loosened.) */ recentXid = ReadNextTransactionId(); - xidForceLimit = recentXid - autovacuum_freeze_max_age; + xidAgeLimit = recentXid - autovacuum_freeze_max_age; /* ensure it's a "normal" XID, else TransactionIdPrecedes misbehaves */ /* this can cause the limit to go backwards by 3, but that's OK */ - if (xidForceLimit < FirstNormalTransactionId) - xidForceLimit -= FirstNormalTransactionId; + if (xidAgeLimit < FirstNormalTransactionId) + xidAgeLimit -= FirstNormalTransactionId; /* Also determine the oldest datminmxid we will consider. */ recentMulti = ReadNextMultiXactId(); - multiForceLimit = recentMulti - MultiXactMemberFreezeThreshold(); - if (multiForceLimit < FirstMultiXactId) - multiForceLimit -= FirstMultiXactId; + multiAgeLimit = recentMulti - MultiXactMemberFreezeThreshold(); + if (multiAgeLimit < FirstMultiXactId) + multiAgeLimit -= FirstMultiXactId; /* * Choose a database to connect to. We pick the database that was least @@ -1229,7 +1231,7 @@ do_start_worker(void) dlist_iter iter; /* Check to see if this one is at risk of wraparound */ - if (TransactionIdPrecedes(tmp->adw_frozenxid, xidForceLimit)) + if (TransactionIdPrecedes(tmp->adw_frozenxid, xidAgeLimit)) { if (avdb == NULL || TransactionIdPrecedes(tmp->adw_frozenxid, @@ -1240,7 +1242,7 @@ do_start_worker(void) } else if (for_xid_wrap) continue; /* ignore not-at-risk DBs */ - else if (MultiXactIdPrecedes(tmp->adw_minmulti, multiForceLimit)) + else if (MultiXactIdPrecedes(tmp->adw_minmulti, multiAgeLimit)) { if (avdb == NULL || MultiXactIdPrecedes(tmp->adw_minmulti, avdb->adw_minmulti)) @@ -1626,7 +1628,7 @@ AutoVacWorkerMain(int argc, char *argv[]) /* * Force synchronous replication off to allow regular maintenance even if * we are waiting for standbys to connect. This is important to ensure we - * aren't blocked from performing anti-wraparound tasks. + * aren't blocked from performing table age tasks. */ if (synchronous_commit > SYNCHRONOUS_COMMIT_LOCAL_FLUSH) SetConfigOption("synchronous_commit", "local", @@ -2767,6 +2769,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, autovac_table *tab = NULL; bool wraparound; AutoVacOpts *avopts; + AutoVacType trigger; /* fetch the relation's relcache entry */ classTup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(relid)); @@ -2790,9 +2793,10 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, avopts = &hentry->ar_reloptions; } - recheck_relation_needs_vacanalyze(relid, avopts, classForm, - effective_multixact_freeze_max_age, - &dovacuum, &doanalyze, &wraparound); + trigger = recheck_relation_needs_vacanalyze(relid, avopts, classForm, + effective_multixact_freeze_max_age, + &dovacuum, &doanalyze, + &wraparound); /* OK, it needs something done */ if (doanalyze || dovacuum) @@ -2878,6 +2882,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, tab->at_params.multixact_freeze_min_age = multixact_freeze_min_age; tab->at_params.multixact_freeze_table_age = multixact_freeze_table_age; tab->at_params.is_wraparound = wraparound; + tab->at_params.trigger = trigger; tab->at_params.log_min_duration = log_min_duration; tab->at_vacuum_cost_limit = vac_cost_limit; tab->at_vacuum_cost_delay = vac_cost_delay; @@ -2906,7 +2911,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, * Fetch the pgstat of a relation and recheck whether a relation * needs to be vacuumed or analyzed. */ -static void +static AutoVacType recheck_relation_needs_vacanalyze(Oid relid, AutoVacOpts *avopts, Form_pg_class classForm, @@ -2916,26 +2921,29 @@ recheck_relation_needs_vacanalyze(Oid relid, bool *wraparound) { PgStat_StatTabEntry *tabentry; + AutoVacType trigger; /* fetch the pgstat table entry */ tabentry = pgstat_fetch_stat_tabentry_ext(classForm->relisshared, relid); - relation_needs_vacanalyze(relid, avopts, classForm, tabentry, - effective_multixact_freeze_max_age, - dovacuum, doanalyze, wraparound); + trigger = relation_needs_vacanalyze(relid, avopts, classForm, tabentry, + effective_multixact_freeze_max_age, + dovacuum, doanalyze, wraparound); /* ignore ANALYZE for toast tables */ if (classForm->relkind == RELKIND_TOASTVALUE) *doanalyze = false; + + return trigger; } /* * relation_needs_vacanalyze * - * Check whether a relation needs to be vacuumed or analyzed; return each into - * "dovacuum" and "doanalyze", respectively. Also return whether the vacuum is - * being forced because of Xid or multixact wraparound. + * Check whether a relation needs to be vacuumed or analyzed; set each using + * "dovacuum" and "doanalyze", respectively. Also indicate whether the vacuum + * must use special antiwraparound protections by setting "wraparound". * * relopts is a pointer to the AutoVacOpts options (either for itself in the * case of a plain table, or for either itself or its parent table in the case @@ -2953,9 +2961,9 @@ recheck_relation_needs_vacanalyze(Oid relid, * the number of tuples (both live and dead) that there were as of the last * analyze. This is asymmetric to the VACUUM case. * - * We also force vacuum if the table's relfrozenxid is more than freeze_max_age - * transactions back, and if its relminmxid is more than - * multixact_freeze_max_age multixacts back. + * We also force table age vacuum if the table's relfrozenxid is more than + * freeze_max_age transactions back, and if its relminmxid is more than + * multixact_freeze_max_age multixacts back. This cannot be disabled. * * A table whose autovacuum_enabled option is false is * automatically skipped (unless we have to vacuum it due to freeze_max_age). @@ -2966,8 +2974,17 @@ recheck_relation_needs_vacanalyze(Oid relid, * autovacuum_vacuum_threshold GUC variable. Similarly, a vac_scale_factor * value < 0 is substituted with the value of * autovacuum_vacuum_scale_factor GUC variable. Ditto for analyze. + * + * Return value is the condition that triggered autovacuum to run VACUUM + * (useful only when *dovacuum is set). There can only be exactly one + * triggering condition, even when multiple thresholds happened to be crossed + * at the same time. We prefer to return "table XID age" in the event of such + * a conflict, after which we prefer "table MXID age" as the criteria, then + * "dead tuples", with "inserted tuples" placed last. These predecence rules + * are largely arbitrary. We must at least ensure that all antiwraparound + * autovacuums are advertised as triggered by table XID/MXID age criteria. */ -static void +static AutoVacType relation_needs_vacanalyze(Oid relid, AutoVacOpts *relopts, Form_pg_class classForm, @@ -2978,7 +2995,10 @@ relation_needs_vacanalyze(Oid relid, bool *doanalyze, bool *wraparound) { - bool force_vacuum; + TransactionId relfrozenxid = classForm->relfrozenxid; + MultiXactId relminmxid = classForm->relminmxid; + AutoVacType trigger = AUTOVACUUM_NONE; + bool tableagevac; bool av_enabled; float4 reltuples; /* pg_class.reltuples */ @@ -3003,8 +3023,8 @@ relation_needs_vacanalyze(Oid relid, /* freeze parameters */ int freeze_max_age; int multixact_freeze_max_age; - TransactionId xidForceLimit; - MultiXactId multiForceLimit; + TransactionId xidAgeLimit; + MultiXactId multiAgeLimit; Assert(classForm != NULL); Assert(OidIsValid(relid)); @@ -3051,40 +3071,49 @@ relation_needs_vacanalyze(Oid relid, av_enabled = (relopts ? relopts->enabled : true); - /* Force vacuum if table is at risk of wraparound */ - xidForceLimit = recentXid - freeze_max_age; - if (xidForceLimit < FirstNormalTransactionId) - xidForceLimit -= FirstNormalTransactionId; - force_vacuum = (TransactionIdIsNormal(classForm->relfrozenxid) && - TransactionIdPrecedes(classForm->relfrozenxid, - xidForceLimit)); - if (!force_vacuum) - { - multiForceLimit = recentMulti - multixact_freeze_max_age; - if (multiForceLimit < FirstMultiXactId) - multiForceLimit -= FirstMultiXactId; - force_vacuum = MultiXactIdIsValid(classForm->relminmxid) && - MultiXactIdPrecedes(classForm->relminmxid, multiForceLimit); - } - *wraparound = force_vacuum; + /* Force vacuum if table age exceeds cutoff */ + xidAgeLimit = recentXid - freeze_max_age; + if (xidAgeLimit < FirstNormalTransactionId) + xidAgeLimit -= FirstNormalTransactionId; + multiAgeLimit = recentMulti - multixact_freeze_max_age; + if (multiAgeLimit < FirstMultiXactId) + multiAgeLimit -= FirstMultiXactId; - /* User disabled it in pg_class.reloptions? (But ignore if at risk) */ - if (!av_enabled && !force_vacuum) + tableagevac = true; + *wraparound = false; + /* See header comments about trigger precedence */ + if (TransactionIdIsNormal(relfrozenxid) && + TransactionIdPrecedes(relfrozenxid, xidAgeLimit)) + trigger = AUTOVACUUM_TABLE_XID_AGE; + else if (MultiXactIdIsValid(relminmxid) && + MultiXactIdPrecedes(relminmxid, multiAgeLimit)) + trigger = AUTOVACUUM_TABLE_MXID_AGE; + else + tableagevac = false; + + /* User disabled non-table-age autovacuums in pg_class.reloptions? */ + if (!av_enabled && !tableagevac) { *doanalyze = false; *dovacuum = false; - return; + return AUTOVACUUM_NONE; } + /* A table age autovacuum always gets antiwraparound protections */ + *wraparound = tableagevac; + /* * If we found stats for the table, and autovacuum is currently enabled, * make a threshold-based decision whether to vacuum and/or analyze. If - * autovacuum is currently disabled, we must be here for anti-wraparound + * autovacuum is currently disabled, we must be here for forced table age * vacuuming only, so don't vacuum (or analyze) anything that's not being * forced. */ if (PointerIsValid(tabentry) && AutoVacuumingActive()) { + bool deadtupvac, + inserttupvac; + reltuples = classForm->reltuples; vactuples = tabentry->dead_tuples; instuples = tabentry->ins_since_vacuum; @@ -3112,25 +3141,39 @@ relation_needs_vacanalyze(Oid relid, NameStr(classForm->relname), vactuples, vacthresh, anltuples, anlthresh); + deadtupvac = (vactuples > vacthresh); + inserttupvac = (vac_ins_base_thresh >= 0 && instuples > vacinsthresh); + /* See header comments about trigger precedence */ + if (!tableagevac) + { + if (deadtupvac) + trigger = AUTOVACUUM_DEAD_TUPLES; + else if (inserttupvac) + trigger = AUTOVACUUM_INSERTED_TUPLES; + } + /* Determine if this table needs vacuum or analyze. */ - *dovacuum = force_vacuum || (vactuples > vacthresh) || - (vac_ins_base_thresh >= 0 && instuples > vacinsthresh); + *dovacuum = (tableagevac || deadtupvac || inserttupvac); *doanalyze = (anltuples > anlthresh); } else { /* * Skip a table not found in stat hash, unless we have to force vacuum - * for anti-wrap purposes. If it's not acted upon, there's no need to + * for table age purposes. If it's not acted upon, there's no need to * vacuum it. */ - *dovacuum = force_vacuum; + *dovacuum = tableagevac; *doanalyze = false; } /* ANALYZE refuses to work with pg_statistic */ if (relid == StatisticRelationId) *doanalyze = false; + + Assert((trigger != AUTOVACUUM_NONE) == *dovacuum); + + return trigger; } /* -- 2.39.0