From 7a1d9cd82e72d3cccb356b8930e32b5154e14e00 Mon Sep 17 00:00:00 2001 From: Reid Thompson Date: Thu, 11 Aug 2022 12:01:25 -0400 Subject: [PATCH 1/2] Add tracking of backend memory allocated to pg_stat_activity This new field displays the current bytes of memory allocated to the backend process. It is updated as memory for the process is palloc'd/pfree'd. Memory allocated to items on the freelist is included in the displayed value. Dynamic shared memory allocations are included only in the value displayed for the backend that created them, they are not included in the value for backends that are attached to them to avoid double counting. On occasion, orphaned memory segments may be cleaned up on postmaster startup. This may result in decreasing the sum without a prior increment. We limit the floor of backend_mem_allocated to zero. Updated pg_stat_activity documentation for the new column. --- doc/src/sgml/monitoring.sgml | 15 ++++ src/backend/catalog/system_views.sql | 1 + src/backend/postmaster/autovacuum.c | 6 ++ src/backend/postmaster/postmaster.c | 13 ++++ src/backend/postmaster/syslogger.c | 3 + src/backend/storage/ipc/dsm_impl.c | 81 +++++++++++++++++++++ src/backend/utils/activity/backend_status.c | 45 ++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 4 +- src/backend/utils/mmgr/aset.c | 17 +++++ src/backend/utils/mmgr/generation.c | 15 ++++ src/backend/utils/mmgr/slab.c | 23 ++++++ src/include/catalog/pg_proc.dat | 6 +- src/include/utils/backend_status.h | 63 +++++++++++++++- src/test/regress/expected/rules.out | 9 ++- src/test/regress/expected/stats.out | 11 +++ src/test/regress/sql/stats.sql | 5 +- 16 files changed, 307 insertions(+), 10 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 6249bb50d0..c1c2eb3531 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -963,6 +963,21 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + + allocated_bytes bigint + + + Memory currently allocated to this backend in bytes. This is the balance + of bytes allocated and freed by this backend. Dynamic shared memory + allocations are included only in the value displayed for the backend that + created them, they are not included in the value for backends that are + attached to them to avoid double counting. Use pg_size_pretty + described in to make this value + more easily readable. + + + query text diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 34ca0e739f..9544e50483 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -863,6 +863,7 @@ CREATE VIEW pg_stat_activity AS S.state, S.backend_xid, s.backend_xmin, + S.allocated_bytes, S.query_id, S.query, S.backend_type diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index ff6149a179..59c9bcf8c4 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -407,6 +407,9 @@ StartAutoVacLauncher(void) #ifndef EXEC_BACKEND case 0: + /* Zero allocated bytes to avoid double counting parent allocation */ + pgstat_zero_my_allocated_bytes(); + /* in postmaster child ... */ InitPostmasterChild(); @@ -1485,6 +1488,9 @@ StartAutoVacWorker(void) #ifndef EXEC_BACKEND case 0: + /* Zero allocated bytes to avoid double counting parent allocation */ + pgstat_zero_my_allocated_bytes(); + /* in postmaster child ... */ InitPostmasterChild(); diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 2552327d90..1f09781be8 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4167,6 +4167,9 @@ BackendStartup(Port *port) { free(bn); + /* Zero allocated bytes to avoid double counting parent allocation */ + pgstat_zero_my_allocated_bytes(); + /* Detangle from postmaster */ InitPostmasterChild(); @@ -5374,6 +5377,11 @@ StartChildProcess(AuxProcType type) MemoryContextDelete(PostmasterContext); PostmasterContext = NULL; + /* Zero allocated bytes to avoid double counting parent allocation. + * Needs to be after the MemoryContextDelete(PostmasterContext) above. + */ + pgstat_zero_my_allocated_bytes(); + AuxiliaryProcessMain(type); /* does not return */ } #endif /* EXEC_BACKEND */ @@ -5767,6 +5775,11 @@ do_start_bgworker(RegisteredBgWorker *rw) MemoryContextDelete(PostmasterContext); PostmasterContext = NULL; + /* Zero allocated bytes to avoid double counting parent allocation. + * Needs to be after the MemoryContextDelete(PostmasterContext) above. + */ + pgstat_zero_my_allocated_bytes(); + StartBackgroundWorker(); exit(1); /* should not get here */ diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index 858a2f6b2b..9081ae140f 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -679,6 +679,9 @@ SysLogger_Start(void) #ifndef EXEC_BACKEND case 0: + /* Zero allocated bytes to avoid double counting parent allocation */ + pgstat_zero_my_allocated_bytes(); + /* in postmaster child ... */ InitPostmasterChild(); diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c index f0965c3481..22885c7bd2 100644 --- a/src/backend/storage/ipc/dsm_impl.c +++ b/src/backend/storage/ipc/dsm_impl.c @@ -66,6 +66,7 @@ #include "postmaster/postmaster.h" #include "storage/dsm_impl.h" #include "storage/fd.h" +#include "utils/backend_status.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -232,6 +233,14 @@ dsm_impl_posix(dsm_op op, dsm_handle handle, Size request_size, name))); return false; } + + /* + * Detach and destroy pass through here, only decrease the memory + * shown allocated in pg_stat_activity when the creator destroys the + * allocation. + */ + if (op == DSM_OP_DESTROY) + pgstat_report_allocated_bytes(*mapped_size, PG_ALLOC_DECREASE); *mapped_address = NULL; *mapped_size = 0; if (op == DSM_OP_DESTROY && shm_unlink(name) != 0) @@ -332,6 +341,36 @@ dsm_impl_posix(dsm_op op, dsm_handle handle, Size request_size, name))); return false; } + + /* + * Attach and create pass through here, only update backend memory + * allocated in pg_stat_activity for the creator process. + */ + if (op == DSM_OP_CREATE) + { + /* + * Posix creation calls dsm_impl_posix_resize implying that resizing + * occurs or may be added in the future. As implemented + * dsm_impl_posix_resize utilizes fallocate or truncate, passing the + * whole new size as input, growing the allocation as needed (only + * truncate supports shrinking). We update by replacing the old + * allocation with the new. + */ +#if defined(HAVE_POSIX_FALLOCATE) && defined(__linux__) + /* + * posix_fallocate does not shrink allocations, adjust only on + * allocation increase. + */ + if (request_size > *mapped_size) + { + pgstat_report_allocated_bytes(request_size - *mapped_size, + PG_ALLOC_INCREASE); + } +#else + pgstat_report_allocated_bytes(*mapped_size, PG_ALLOC_DECREASE); + pgstat_report_allocated_bytes(request_size, PG_ALLOC_INCREASE); +#endif + } *mapped_address = address; *mapped_size = request_size; close(fd); @@ -537,6 +576,14 @@ dsm_impl_sysv(dsm_op op, dsm_handle handle, Size request_size, name))); return false; } + + /* + * Detach and destroy pass through here, only decrease the memory + * shown allocated in pg_stat_activity when the creator destroys the + * allocation. + */ + if (op == DSM_OP_DESTROY) + pgstat_report_allocated_bytes(*mapped_size, PG_ALLOC_DECREASE); *mapped_address = NULL; *mapped_size = 0; if (op == DSM_OP_DESTROY && shmctl(ident, IPC_RMID, NULL) < 0) @@ -584,6 +631,13 @@ dsm_impl_sysv(dsm_op op, dsm_handle handle, Size request_size, name))); return false; } + + /* + * Attach and create pass through here, only update backend memory + * allocated in pg_stat_activity for the creator process. + */ + if (op == DSM_OP_CREATE) + pgstat_report_allocated_bytes(request_size, PG_ALLOC_INCREASE); *mapped_address = address; *mapped_size = request_size; @@ -652,6 +706,13 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size, return false; } + /* + * Detach and destroy pass through here, only decrease the memory + * shown allocated in pg_stat_activity when the creator destroys the + * allocation. + */ + if (op == DSM_OP_DESTROY) + pgstat_report_allocated_bytes(*mapped_size, PG_ALLOC_DECREASE); *impl_private = NULL; *mapped_address = NULL; *mapped_size = 0; @@ -768,6 +829,12 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size, return false; } + /* + * Attach and create pass through here, only update backend memory + * allocated in pg_stat_activity for the creator process. + */ + if (op == DSM_OP_CREATE) + pgstat_report_allocated_bytes(info.RegionSize, PG_ALLOC_INCREASE); *mapped_address = address; *mapped_size = info.RegionSize; *impl_private = hmap; @@ -812,6 +879,13 @@ dsm_impl_mmap(dsm_op op, dsm_handle handle, Size request_size, name))); return false; } + + /* + * Detach and destroy pass through here, only decrease the memory + * shown allocated in pg_stat_activity when the creator destroys the + * allocation. + */ + pgstat_report_allocated_bytes(*mapped_size, PG_ALLOC_DECREASE); *mapped_address = NULL; *mapped_size = 0; if (op == DSM_OP_DESTROY && unlink(name) != 0) @@ -933,6 +1007,13 @@ dsm_impl_mmap(dsm_op op, dsm_handle handle, Size request_size, name))); return false; } + + /* + * Attach and create pass through here, only update backend memory + * allocated in pg_stat_activity for the creator process. + */ + if (op == DSM_OP_CREATE) + pgstat_report_allocated_bytes(request_size, PG_ALLOC_INCREASE); *mapped_address = address; *mapped_size = request_size; diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 608d01ea0d..7baf2db57d 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -49,6 +49,9 @@ int pgstat_track_activity_query_size = 1024; /* exposed so that backend_progress.c can access it */ PgBackendStatus *MyBEEntry = NULL; +/* Memory allocated to this backend prior to pgstats initialization */ +uint64 local_my_allocated_bytes = 0; +uint64 *my_allocated_bytes = &local_my_allocated_bytes; static PgBackendStatus *BackendStatusArray = NULL; static char *BackendAppnameBuffer = NULL; @@ -400,6 +403,15 @@ pgstat_bestart(void) lbeentry.st_progress_command_target = InvalidOid; lbeentry.st_query_id = UINT64CONST(0); + /* Alter allocation reporting from local_my_allocated_bytes to shared memory */ + pgstat_set_allocated_bytes_storage(&MyBEEntry->allocated_bytes); + + /* Populate sum of memory allocated prior to pgstats initialization to pgstats + * and zero the local variable. + */ + lbeentry.allocated_bytes += local_my_allocated_bytes; + local_my_allocated_bytes = 0; + /* * we don't zero st_progress_param here to save cycles; nobody should * examine it until st_progress_command has been set to something other @@ -459,6 +471,11 @@ pgstat_beshutdown_hook(int code, Datum arg) { volatile PgBackendStatus *beentry = MyBEEntry; + /* + * Stop reporting memory allocation changes to &MyBEEntry->allocated_bytes + */ + pgstat_reset_allocated_bytes_storage(); + /* * Clear my status entry, following the protocol of bumping st_changecount * before and after. We use a volatile pointer here to ensure the @@ -1194,3 +1211,31 @@ pgstat_clip_activity(const char *raw_activity) return activity; } + +/* + * Configure bytes allocated reporting to report allocated bytes to + * *allocated_bytes. *allocated_bytes needs to be valid until + * pgstat_set_allocated_bytes_storage() is called. + * + * Expected to be called during backend startup (in pgstat_bestart), to point + * my_allocated_bytes into shared memory. + */ +void +pgstat_set_allocated_bytes_storage(uint64 *new_allocated_bytes) +{ + my_allocated_bytes = new_allocated_bytes; + *new_allocated_bytes = local_my_allocated_bytes; +} + +/* + * Reset allocated bytes storage location. + * + * Expected to be called during backend shutdown, before the location set up + * by pgstat_set_allocated_bytes_storage() becomes invalid. + */ +void +pgstat_reset_allocated_bytes_storage(void) +{ + my_allocated_bytes = &local_my_allocated_bytes; +} + diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index b61a12382b..35fab203d4 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -303,7 +303,7 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) Datum pg_stat_get_activity(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_ACTIVITY_COLS 30 +#define PG_STAT_GET_ACTIVITY_COLS 31 int num_backends = pgstat_fetch_stat_numbackends(); int curr_backend; int pid = PG_ARGISNULL(0) ? -1 : PG_GETARG_INT32(0); @@ -359,6 +359,8 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) else nulls[16] = true; + values[30] = UInt64GetDatum(beentry->allocated_bytes); + /* Values only available to role member or pg_read_all_stats */ if (HAS_PGSTAT_PERMISSIONS(beentry->st_userid)) { diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c index 2589941ec4..1a2d86239c 100644 --- a/src/backend/utils/mmgr/aset.c +++ b/src/backend/utils/mmgr/aset.c @@ -47,6 +47,7 @@ #include "postgres.h" #include "port/pg_bitutils.h" +#include "utils/backend_status.h" #include "utils/memdebug.h" #include "utils/memutils.h" #include "utils/memutils_memorychunk.h" @@ -521,6 +522,7 @@ AllocSetContextCreateInternal(MemoryContext parent, name); ((MemoryContext) set)->mem_allocated = firstBlockSize; + pgstat_report_allocated_bytes(firstBlockSize, PG_ALLOC_INCREASE); return (MemoryContext) set; } @@ -543,6 +545,7 @@ AllocSetReset(MemoryContext context) AllocSet set = (AllocSet) context; AllocBlock block; Size keepersize PG_USED_FOR_ASSERTS_ONLY; + uint64 deallocation = 0; Assert(AllocSetIsValid(set)); @@ -585,6 +588,7 @@ AllocSetReset(MemoryContext context) { /* Normal case, release the block */ context->mem_allocated -= block->endptr - ((char *) block); + deallocation += block->endptr - ((char *) block); #ifdef CLOBBER_FREED_MEMORY wipe_mem(block, block->freeptr - ((char *) block)); @@ -595,6 +599,7 @@ AllocSetReset(MemoryContext context) } Assert(context->mem_allocated == keepersize); + pgstat_report_allocated_bytes(deallocation, PG_ALLOC_DECREASE); /* Reset block size allocation sequence, too */ set->nextBlockSize = set->initBlockSize; @@ -613,6 +618,7 @@ AllocSetDelete(MemoryContext context) AllocSet set = (AllocSet) context; AllocBlock block = set->blocks; Size keepersize PG_USED_FOR_ASSERTS_ONLY; + uint64 deallocation = 0; Assert(AllocSetIsValid(set)); @@ -651,11 +657,13 @@ AllocSetDelete(MemoryContext context) freelist->first_free = (AllocSetContext *) oldset->header.nextchild; freelist->num_free--; + deallocation += oldset->header.mem_allocated; /* All that remains is to free the header/initial block */ free(oldset); } Assert(freelist->num_free == 0); + pgstat_report_allocated_bytes(deallocation, PG_ALLOC_DECREASE); } /* Now add the just-deleted context to the freelist. */ @@ -672,7 +680,10 @@ AllocSetDelete(MemoryContext context) AllocBlock next = block->next; if (block != set->keeper) + { context->mem_allocated -= block->endptr - ((char *) block); + deallocation += block->endptr - ((char *) block); + } #ifdef CLOBBER_FREED_MEMORY wipe_mem(block, block->freeptr - ((char *) block)); @@ -685,6 +696,7 @@ AllocSetDelete(MemoryContext context) } Assert(context->mem_allocated == keepersize); + pgstat_report_allocated_bytes(deallocation + context->mem_allocated, PG_ALLOC_DECREASE); /* Finally, free the context header, including the keeper block */ free(set); @@ -734,6 +746,7 @@ AllocSetAlloc(MemoryContext context, Size size) return NULL; context->mem_allocated += blksize; + pgstat_report_allocated_bytes(blksize, PG_ALLOC_INCREASE); block->aset = set; block->freeptr = block->endptr = ((char *) block) + blksize; @@ -944,6 +957,7 @@ AllocSetAlloc(MemoryContext context, Size size) return NULL; context->mem_allocated += blksize; + pgstat_report_allocated_bytes(blksize, PG_ALLOC_INCREASE); block->aset = set; block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; @@ -1041,6 +1055,7 @@ AllocSetFree(void *pointer) block->next->prev = block->prev; set->header.mem_allocated -= block->endptr - ((char *) block); + pgstat_report_allocated_bytes(block->endptr - ((char *) block), PG_ALLOC_DECREASE); #ifdef CLOBBER_FREED_MEMORY wipe_mem(block, block->freeptr - ((char *) block)); @@ -1171,7 +1186,9 @@ AllocSetRealloc(void *pointer, Size size) /* updated separately, not to underflow when (oldblksize > blksize) */ set->header.mem_allocated -= oldblksize; + pgstat_report_allocated_bytes(oldblksize, PG_ALLOC_DECREASE); set->header.mem_allocated += blksize; + pgstat_report_allocated_bytes(blksize, PG_ALLOC_INCREASE); block->freeptr = block->endptr = ((char *) block) + blksize; diff --git a/src/backend/utils/mmgr/generation.c b/src/backend/utils/mmgr/generation.c index ebcb61e9b6..b06fb0c6a4 100644 --- a/src/backend/utils/mmgr/generation.c +++ b/src/backend/utils/mmgr/generation.c @@ -37,6 +37,7 @@ #include "lib/ilist.h" #include "port/pg_bitutils.h" +#include "utils/backend_status.h" #include "utils/memdebug.h" #include "utils/memutils.h" #include "utils/memutils_memorychunk.h" @@ -267,6 +268,7 @@ GenerationContextCreate(MemoryContext parent, name); ((MemoryContext) set)->mem_allocated = firstBlockSize; + pgstat_report_allocated_bytes(firstBlockSize, PG_ALLOC_INCREASE); return (MemoryContext) set; } @@ -283,6 +285,7 @@ GenerationReset(MemoryContext context) { GenerationContext *set = (GenerationContext *) context; dlist_mutable_iter miter; + uint64 deallocation = 0; Assert(GenerationIsValid(set)); @@ -305,9 +308,14 @@ GenerationReset(MemoryContext context) if (block == set->keeper) GenerationBlockMarkEmpty(block); else + { + deallocation += block->blksize; GenerationBlockFree(set, block); + } } + pgstat_report_allocated_bytes(deallocation, PG_ALLOC_DECREASE); + /* set it so new allocations to make use of the keeper block */ set->block = set->keeper; @@ -328,6 +336,9 @@ GenerationDelete(MemoryContext context) { /* Reset to release all releasable GenerationBlocks */ GenerationReset(context); + + pgstat_report_allocated_bytes(context->mem_allocated, PG_ALLOC_DECREASE); + /* And free the context header and keeper block */ free(context); } @@ -374,6 +385,7 @@ GenerationAlloc(MemoryContext context, Size size) return NULL; context->mem_allocated += blksize; + pgstat_report_allocated_bytes(blksize, PG_ALLOC_INCREASE); /* block with a single (used) chunk */ block->context = set; @@ -477,6 +489,7 @@ GenerationAlloc(MemoryContext context, Size size) return NULL; context->mem_allocated += blksize; + pgstat_report_allocated_bytes(blksize, PG_ALLOC_INCREASE); /* initialize the new block */ GenerationBlockInit(set, block, blksize); @@ -729,6 +742,8 @@ GenerationFree(void *pointer) dlist_delete(&block->node); set->header.mem_allocated -= block->blksize; + pgstat_report_allocated_bytes(block->blksize, PG_ALLOC_DECREASE); + free(block); } diff --git a/src/backend/utils/mmgr/slab.c b/src/backend/utils/mmgr/slab.c index 33dca0f37c..15d3380640 100644 --- a/src/backend/utils/mmgr/slab.c +++ b/src/backend/utils/mmgr/slab.c @@ -69,6 +69,7 @@ #include "postgres.h" #include "lib/ilist.h" +#include "utils/backend_status.h" #include "utils/memdebug.h" #include "utils/memutils.h" #include "utils/memutils_memorychunk.h" @@ -413,6 +414,13 @@ SlabContextCreate(MemoryContext parent, parent, name); + /* + * If SlabContextCreate is updated to add context header size to + * context->mem_allocated, then update here and SlabDelete appropriately + */ + pgstat_report_allocated_bytes(Slab_CONTEXT_HDRSZ(slab->chunksPerBlock), + PG_ALLOC_INCREASE); + return (MemoryContext) slab; } @@ -429,6 +437,7 @@ SlabReset(MemoryContext context) SlabContext *slab = (SlabContext *) context; dlist_mutable_iter miter; int i; + uint64 deallocation = 0; Assert(SlabIsValid(slab)); @@ -449,6 +458,7 @@ SlabReset(MemoryContext context) #endif free(block); context->mem_allocated -= slab->blockSize; + deallocation += slab->blockSize; } /* walk over blocklist and free the blocks */ @@ -465,9 +475,11 @@ SlabReset(MemoryContext context) #endif free(block); context->mem_allocated -= slab->blockSize; + deallocation += slab->blockSize; } } + pgstat_report_allocated_bytes(deallocation, PG_ALLOC_DECREASE); slab->curBlocklistIndex = 0; Assert(context->mem_allocated == 0); @@ -480,8 +492,17 @@ SlabReset(MemoryContext context) void SlabDelete(MemoryContext context) { + /* Reset to release all the SlabBlocks */ SlabReset(context); + + /* + * Until context header allocation is included in context->mem_allocated, + * cast to slab and decrement the header allocation + */ + pgstat_report_allocated_bytes(Slab_CONTEXT_HDRSZ(((SlabContext *)context)->chunksPerBlock), + PG_ALLOC_DECREASE); + /* And free the context header */ free(context); } @@ -546,6 +567,7 @@ SlabAlloc(MemoryContext context, Size size) block->slab = slab; context->mem_allocated += slab->blockSize; + pgstat_report_allocated_bytes(slab->blockSize, PG_ALLOC_INCREASE); /* use the first chunk in the new block */ chunk = SlabBlockGetChunk(slab, block, 0); @@ -732,6 +754,7 @@ SlabFree(void *pointer) #endif free(block); slab->header.mem_allocated -= slab->blockSize; + pgstat_report_allocated_bytes(slab->blockSize, PG_ALLOC_DECREASE); } /* diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 505595620e..cd3896869e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5404,9 +5404,9 @@ proname => 'pg_stat_get_activity', prorows => '100', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'int4', - proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,int4,int8}', - proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,leader_pid,query_id}', + proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,int4,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,leader_pid,query_id,allocated_bytes}', prosrc => 'pg_stat_get_activity' }, { oid => '3318', descr => 'statistics: information about progress of backends running maintenance command', diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h index f7bd83113a..754ff0dc62 100644 --- a/src/include/utils/backend_status.h +++ b/src/include/utils/backend_status.h @@ -15,6 +15,7 @@ #include "miscadmin.h" /* for BackendType */ #include "storage/backendid.h" #include "utils/backend_progress.h" +#include "common/int.h" /* ---------- @@ -32,6 +33,13 @@ typedef enum BackendState STATE_DISABLED } BackendState; +/* Enum helper for reporting memory allocated bytes */ +enum allocation_direction +{ + PG_ALLOC_DECREASE = -1, + PG_ALLOC_IGNORE, + PG_ALLOC_INCREASE, +}; /* ---------- * Shared-memory data structures @@ -169,6 +177,9 @@ typedef struct PgBackendStatus /* query identifier, optionally computed using post_parse_analyze_hook */ uint64 st_query_id; + + /* Current memory allocated to this backend */ + uint64 allocated_bytes; } PgBackendStatus; @@ -293,6 +304,7 @@ extern PGDLLIMPORT int pgstat_track_activity_query_size; * ---------- */ extern PGDLLIMPORT PgBackendStatus *MyBEEntry; +extern PGDLLIMPORT uint64 *my_allocated_bytes; /* ---------- @@ -324,7 +336,8 @@ extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser); extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer, int buflen); extern uint64 pgstat_get_my_query_id(void); - +extern void pgstat_set_allocated_bytes_storage(uint64 *allocated_bytes); +extern void pgstat_reset_allocated_bytes_storage(void); /* ---------- * Support functions for the SQL-callable functions to @@ -336,5 +349,53 @@ extern PgBackendStatus *pgstat_fetch_stat_beentry(BackendId beid); extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry(int beid); extern char *pgstat_clip_activity(const char *raw_activity); +/* ---------- + * pgstat_report_allocated_bytes() - + * + * Called to report change in memory allocated for this backend. + * + * my_allocated_bytes initially points to local memory, making it safe to call + * this before pgstats has been initialized. allocation_direction is a + * positive/negative multiplier enum defined above. + * ---------- + */ +static inline void +pgstat_report_allocated_bytes(int64 allocated_bytes, int allocation_direction) +{ + uint64 temp; + + /* + * Avoid *my_allocated_bytes unsigned integer overflow on + * PG_ALLOC_DECREASE + */ + if (allocation_direction == PG_ALLOC_DECREASE && + pg_sub_u64_overflow(*my_allocated_bytes, allocated_bytes, &temp)) + { + *my_allocated_bytes = 0; + ereport(LOG, + errmsg("Backend %d deallocated %lld bytes, exceeding the %llu bytes it is currently reporting allocated. Setting reported to 0.", + MyProcPid, (long long) allocated_bytes, + (unsigned long long) *my_allocated_bytes)); + } + else + *my_allocated_bytes += (allocated_bytes) * allocation_direction; + + return; +} + +/* --------- + * pgstat_zero_my_allocated_bytes() - + * + * Called to zero out local allocated bytes variable after fork to avoid double + * counting allocations. + * --------- + */ +static inline void +pgstat_zero_my_allocated_bytes(void) +{ + *my_allocated_bytes = 0; + + return; +} #endif /* BACKEND_STATUS_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index e953d1f515..271648619a 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1756,10 +1756,11 @@ pg_stat_activity| SELECT s.datid, s.state, s.backend_xid, s.backend_xmin, + s.allocated_bytes, s.query_id, s.query, s.backend_type - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes) LEFT JOIN pg_database d ON ((s.datid = d.oid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_all_indexes| SELECT c.oid AS relid, @@ -1874,7 +1875,7 @@ pg_stat_gssapi| SELECT pid, gss_auth AS gss_authenticated, gss_princ AS principal, gss_enc AS encrypted - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id) + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes) WHERE (client_port IS NOT NULL); pg_stat_io| SELECT backend_type, io_object, @@ -2067,7 +2068,7 @@ pg_stat_replication| SELECT s.pid, w.sync_priority, w.sync_state, w.reply_time - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes) JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_replication_slots| SELECT s.slot_name, @@ -2101,7 +2102,7 @@ pg_stat_ssl| SELECT pid, ssl_client_dn AS client_dn, ssl_client_serial AS client_serial, ssl_issuer_dn AS issuer_dn - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id) + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes) WHERE (client_port IS NOT NULL); pg_stat_subscription| SELECT su.oid AS subid, su.subname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 937b2101b3..2a848c02a1 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1354,4 +1354,15 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +-- ensure that allocated_bytes exist for backends +SELECT allocated_bytes > 0 AS result FROM pg_stat_activity WHERE backend_type +IN ('checkpointer', 'background writer', 'walwriter', 'autovacuum launcher'); + result +-------- + t + t + t + t +(4 rows) + -- End of Stats Test diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 74e592aa8a..568560c361 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -535,7 +535,6 @@ SET enable_seqscan TO on; SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); - -- Test that the following operations are tracked in pg_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage @@ -678,4 +677,8 @@ SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +-- ensure that allocated_bytes exist for backends +SELECT allocated_bytes > 0 AS result FROM pg_stat_activity WHERE backend_type +IN ('checkpointer', 'background writer', 'walwriter', 'autovacuum launcher'); + -- End of Stats Test -- 2.25.1