From 55efa1c818ac990f759c5fa00d1018f2c9dc7ab0 Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Fri, 8 Dec 2023 13:54:01 -0500 Subject: [PATCH v2 02/10] WIP: Add LSNTimeline to estimate LSN consumption rate Add a timeline of LSN to time mappings to WAL statistics. This allows translation from an LSN <-> time. This commit does not add any users inserting into the timeline or using it to translate time to LSNs. This is very WIP. - It uses a builtin gcc function (so, not portable). - global LSNTimeline added to PgStat_WalStats but not sure if that's right. I didn't add the timeline to PendingWalStats since I wasn't sure we should keep it in PgStat_WalStats - The commit doesn't add any users which insert to the timeline or use it to translate a time to an LSN or vice versa. I do this in heap_vacuum_rel() in later commits, but I'm not sure that makes sense for more general use cases. --- src/backend/utils/activity/pgstat_wal.c | 99 +++++++++++++++++++++++++ src/include/pgstat.h | 45 +++++++++++ src/tools/pgindent/typedefs.list | 2 + 3 files changed, 146 insertions(+) diff --git a/src/backend/utils/activity/pgstat_wal.c b/src/backend/utils/activity/pgstat_wal.c index 6a81b78135..a5b68e4a4d 100644 --- a/src/backend/utils/activity/pgstat_wal.c +++ b/src/backend/utils/activity/pgstat_wal.c @@ -17,8 +17,11 @@ #include "postgres.h" +#include "access/xlog.h" #include "utils/pgstat_internal.h" #include "executor/instrument.h" +#include "utils/builtins.h" +#include "utils/timestamp.h" PgStat_PendingWalStats PendingWalStats = {0}; @@ -184,3 +187,99 @@ pgstat_wal_snapshot_cb(void) sizeof(pgStatLocal.snapshot.wal)); LWLockRelease(&stats_shmem->lock); } + +/* + * Set *a to be the earlier of *a or *b. + */ +static void +lsntime_absorb(LSNTime *a, const LSNTime *b) +{ + LSNTime result; + + if (a->time < b->time) + result = *a; + else if (b->time < a->time) + result = *b; + else if (a->lsn < b->lsn) + result = *a; + else if (b->lsn < a->lsn) + result = *b; + else + result = *a; + + *a = result; +} + +void +lsntime_insert(LSNTimeline *timeline, TimestampTz time, + XLogRecPtr lsn) +{ + LSNTime entrant = {.lsn = lsn,.time = time}; + int buckets = lsn_buckets(timeline); + + if (timeline->members == 0) + { + timeline->data[0] = entrant; + goto done; + } + + for (int i = 0; i < buckets; i++) + { + LSNTime old; + uint64 isset; + + isset = (timeline->members >> (buckets - i - 1)) & 1; + + if (!isset) + { + lsntime_absorb(&timeline->data[i], &entrant); + goto done; + } + + old = timeline->data[i]; + timeline->data[i] = entrant; + entrant = old; + } + + timeline->data[buckets] = entrant; + +done: + timeline->members++; +} + + +XLogRecPtr +estimate_lsn_at_time(const LSNTimeline *timeline, TimestampTz time) +{ + TimestampTz time_elapsed; + XLogRecPtr lsns_elapsed; + double result; + + LSNTime start = {.time = PgStartTime,.lsn = PgStartLSN}; + LSNTime end = {.time = GetCurrentTimestamp(),.lsn = GetXLogInsertRecPtr()}; + + if (time >= end.time) + return end.lsn; + + for (int i = 0; i < lsn_buckets(timeline); i++) + { + if (timeline->data[i].time > time) + continue; + + start = timeline->data[i]; + if (i > 0) + end = timeline->data[i - 1]; + break; + } + + time_elapsed = end.time - start.time; + Assert(time_elapsed != 0); + + lsns_elapsed = end.lsn - start.lsn; + Assert(lsns_elapsed != 0); + + result = (double) (time - start.time) / time_elapsed * lsns_elapsed + start.lsn; + if (result < 0) + return InvalidXLogRecPtr; + return result; +} diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f95d8db0c4..69cea6c48b 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -11,6 +11,10 @@ #ifndef PGSTAT_H #define PGSTAT_H +#include +#include + +#include "access/xlogdefs.h" #include "datatype/timestamp.h" #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ @@ -425,6 +429,43 @@ typedef struct PgStat_StatTabEntry PgStat_Counter autoanalyze_count; } PgStat_StatTabEntry; +/* A time and the insert_lsn recorded at that time. */ +typedef struct LSNTime +{ + TimestampTz time; + XLogRecPtr lsn; +} LSNTime; + +/* + * A timeline of points each consisting of a time and an LSN. An LSN + * consumption rate calculated from the timeline can be used to translate time + * to LSNs and LSNs to time. Points are combined at an increasing rate as they + * become older. Each entry in data is a bucket into which one or more LSNTimes + * have been absorbed. Each bucket can hold twice the members as the preceding + * bucket. + */ +typedef struct LSNTimeline +{ + uint64 members; + LSNTime data[sizeof(uint64) * CHAR_BIT]; +} LSNTimeline; + +/* + * The number of buckets currently in use by the timeline + * MTODO: portability + */ +static inline unsigned int +lsn_buckets(const LSNTimeline *line) +{ + return sizeof(line->members) * CHAR_BIT - __builtin_clzl(line->members); +} + +/* + * MTODO: Add LSNTimeline to PgStat_PendingWalStats and add logic to flush it + * to PgStat_WalStats and check for it in has_pending_wal_stats. I didn't do + * this in this version because efficiently combining two timelines is a bit + * tricky. + */ typedef struct PgStat_WalStats { PgStat_Counter wal_records; @@ -435,6 +476,7 @@ typedef struct PgStat_WalStats PgStat_Counter wal_sync; PgStat_Counter wal_write_time; PgStat_Counter wal_sync_time; + LSNTimeline timeline; TimestampTz stat_reset_timestamp; } PgStat_WalStats; @@ -717,6 +759,9 @@ extern void pgstat_execute_transactional_drops(int ndrops, struct xl_xact_stats_ extern void pgstat_report_wal(bool force); extern PgStat_WalStats *pgstat_fetch_stat_wal(void); +extern XLogRecPtr estimate_lsn_at_time(const LSNTimeline *timeline, TimestampTz time); +extern void lsntime_insert(LSNTimeline *timeline, TimestampTz time, XLogRecPtr lsn); + /* * Variables in pgstat.c diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index d659adbfd6..3a69023dd8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1516,6 +1516,8 @@ LogicalTapeSet LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus +LSNTime +LSNTimeline LtreeGistOptions LtreeSignature MAGIC -- 2.37.2