From 56bfdd6d7296397a7b3f0b282e239fdc86d6580d Mon Sep 17 00:00:00 2001 From: Alexandre Felipe Date: Fri, 6 Mar 2026 17:15:44 +0000 Subject: [PATCH 4/5] Compact PrivateRefCountEntry The previous implementation used an 8-bite (64-bit) entry to store a uint32 count and an uint32 lock mode. That is fine when we store the data separate from the key (buffer). But in the simplehash {key, value} are stored together, so each entry is 12-bytes. This is somewhat awkward as we have to either pad the entry to 16-bytes, or the access will be an alternating aligned/misaligned addreses. However, we are probably not going to need even 16-bits for the count and 2-bits is enough for the lock mode. So we can easily pack these int to a single uint32. Incrementing/decrementing the count continue the same, just using 4 instead of 1, lock mode access will require one or two additional bitwise operations. No bit-shifts are required. --- src/backend/storage/buffer/buf_refcount.c | 167 +++++++++------------- 1 file changed, 70 insertions(+), 97 deletions(-) diff --git a/src/backend/storage/buffer/buf_refcount.c b/src/backend/storage/buffer/buf_refcount.c index ff37355a61e..29dfb720997 100644 --- a/src/backend/storage/buffer/buf_refcount.c +++ b/src/backend/storage/buffer/buf_refcount.c @@ -40,53 +40,54 @@ */ #include "postgres.h" -#include "common/hashfn.h" #include "storage/buf_internals.h" #include "storage/buf_refcount.h" #include "storage/proc.h" -typedef struct PrivateRefCountData +/* + * Implementation details - opaque to callers. + * Packed refcount and lockmode in a single uint32: + * Bits 0-1: lockmode (4 values: UNLOCK, SHARE, SHARE_EXCLUSIVE, EXCLUSIVE) + * Bits 2-31: refcount (30 bits, max ~1 billion) + */ +struct PrivateRefCountEntry { - int32 refcount; - BufferLockMode lockmode; -} PrivateRefCountData; + Buffer buffer; + uint32 data; +}; -struct PrivateRefCountEntry +#define PRIVATEREFCOUNT_LOCKMODE_MASK 0x3 +#define ONE_PRIVATE_REFERENCE 4 /* 1 << 2 */ + +#define PrivateRefCountGetLockMode(d) ((BufferLockMode)((d) & PRIVATEREFCOUNT_LOCKMODE_MASK)) +#define PrivateRefCountSetLockMode(d, m) ((d) = ((d) & ~PRIVATEREFCOUNT_LOCKMODE_MASK) | (m)) +#define PrivateRefCountGetRefCount(d) ((int32)((d) >> 2)) +#define PrivateRefCountIsZero(d) ((d) < ONE_PRIVATE_REFERENCE) + +struct PrivateRefCountIterator { - Buffer buffer; /* hash key - InvalidBuffer means empty */ - PrivateRefCountData data; + int array_index; + bool in_hash; + void *hash_status; }; -/* - * Define simplehash parameters for the overflow hash table. - * We use buffer == InvalidBuffer as the "empty" marker to avoid needing - * a separate status field. - */ +/* Define simplehash for private refcount overflow hash table */ #define SH_PREFIX refcount #define SH_ELEMENT_TYPE PrivateRefCountEntry #define SH_KEY_TYPE Buffer #define SH_KEY buffer -#define SH_HASH_KEY(tb, key) murmurhash32(key) +#define SH_HASH_KEY(tb, key) ((uint32) (key)) #define SH_EQUAL(tb, a, b) ((a) == (b)) #define SH_SCOPE static inline -#define SH_DEFINE -#define SH_DECLARE -/* Use buffer field as empty marker - no separate status needed */ #define SH_ENTRY_EMPTY(entry) ((entry)->buffer == InvalidBuffer) #define SH_MAKE_EMPTY(entry) ((entry)->buffer = InvalidBuffer) -#define SH_MAKE_IN_USE(entry) ((void)0) /* key assignment handles this */ +#define SH_MAKE_IN_USE(entry) ((void) 0) +#define SH_DECLARE +#define SH_DEFINE #include "lib/simplehash.h" -struct PrivateRefCountIterator -{ - int array_index; - bool in_hash; - refcount_iterator hash_iter; - bool hash_iter_valid; -}; - /* Private refcount array and keys */ #define REFCOUNT_ARRAY_ENTRIES 8 static Buffer PrivateRefCountArrayKeys[REFCOUNT_ARRAY_ENTRIES]; @@ -113,7 +114,7 @@ static uint32 MaxProportionalPins = 0; /* Forward declarations */ static void ReservePrivateRefCountEntry(void); static PrivateRefCountEntry *NewPrivateRefCountEntry(Buffer buffer); -static pg_noinline PrivateRefCountEntry *GetPrivateRefCountEntrySlow(Buffer buffer, bool do_move); +static pg_noinline PrivateRefCountEntry *GetPrivateRefCountEntrySlow(Buffer buffer); /* * Initialize private refcount tracking for this backend. @@ -132,7 +133,7 @@ InitPrivateRefCount(void) memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray)); memset(&PrivateRefCountArrayKeys, 0, sizeof(PrivateRefCountArrayKeys)); - PrivateRefCountHash = refcount_create(CurrentMemoryContext, 16, NULL); + PrivateRefCountHash = refcount_create(CurrentMemoryContext, 64, NULL); } /* @@ -158,6 +159,11 @@ ReservePrivateRefCountEntry(void) if (PrivateRefCountArrayKeys[i] == InvalidBuffer) { ReservedRefCountSlot = i; + + /* + * We could return immediately, but iterating till the end of + * the array allows compiler-autovectorization. + */ } } @@ -195,10 +201,7 @@ ReservePrivateRefCountEntry(void) /* clear the now free array slot */ PrivateRefCountArrayKeys[victim_slot] = InvalidBuffer; victim_entry->buffer = InvalidBuffer; - - memset(&victim_entry->data, 0, sizeof(victim_entry->data)); - victim_entry->data.refcount = 0; - victim_entry->data.lockmode = BUFFER_LOCK_UNLOCK; + victim_entry->data = 0; PrivateRefCountOverflowed++; } @@ -221,8 +224,7 @@ NewPrivateRefCountEntry(Buffer buffer) /* and fill it */ PrivateRefCountArrayKeys[ReservedRefCountSlot] = buffer; res->buffer = buffer; - res->data.refcount = 0; - res->data.lockmode = BUFFER_LOCK_UNLOCK; + res->data = 0; /* update cache for the next lookup */ PrivateRefCountEntryLast = ReservedRefCountSlot; @@ -236,7 +238,7 @@ NewPrivateRefCountEntry(Buffer buffer) * Slow-path for GetSharedBufferEntry(). */ static pg_noinline PrivateRefCountEntry * -GetPrivateRefCountEntrySlow(Buffer buffer, bool do_move) +GetPrivateRefCountEntrySlow(Buffer buffer) { PrivateRefCountEntry *res; int match = -1; @@ -266,41 +268,11 @@ GetPrivateRefCountEntrySlow(Buffer buffer, bool do_move) return NULL; res = refcount_lookup(PrivateRefCountHash, buffer); - - if (res == NULL) - return NULL; - else if (!do_move) - { - return res; - } - else - { - /* move buffer from hashtable into the free array slot */ - PrivateRefCountEntry *free; - - ReservePrivateRefCountEntry(); - - Assert(ReservedRefCountSlot != -1); - free = &PrivateRefCountArray[ReservedRefCountSlot]; - Assert(free->buffer == InvalidBuffer); - - free->buffer = buffer; - free->data = res->data; - PrivateRefCountArrayKeys[ReservedRefCountSlot] = buffer; - PrivateRefCountEntryLast = ReservedRefCountSlot; - - ReservedRefCountSlot = -1; - - refcount_delete_item(PrivateRefCountHash, res); - Assert(PrivateRefCountOverflowed > 0); - PrivateRefCountOverflowed--; - - return free; - } + return res; } /* - * Return the PrivateRefCountEntry for the passed buffer. + * Return the PrivateRefCount entry for the passed buffer. * Returns NULL if the buffer is not currently pinned. */ PrivateRefCountEntry * @@ -316,7 +288,7 @@ GetSharedBufferEntry(Buffer buffer) return &PrivateRefCountArray[PrivateRefCountEntryLast]; } - return GetPrivateRefCountEntrySlow(buffer, false); + return GetPrivateRefCountEntrySlow(buffer); } /* @@ -332,25 +304,20 @@ SharedBufferRef(Buffer buffer) Assert(BufferIsValid(buffer)); Assert(!BufferIsLocal(buffer)); - /* Check cache first, then slow path */ - if (likely(PrivateRefCountEntryLast != -1) && - likely(PrivateRefCountArray[PrivateRefCountEntryLast].buffer == buffer)) - { - ref = &PrivateRefCountArray[PrivateRefCountEntryLast]; - } - else - { - ref = GetPrivateRefCountEntrySlow(buffer, true); - } + ref = GetSharedBufferEntry(buffer); if (ref == NULL) { /* New pin - create entry */ ReservePrivateRefCountEntry(); ref = NewPrivateRefCountEntry(buffer); + ref->data = ONE_PRIVATE_REFERENCE; + } + else + { + /* Already pinned - increment */ + ref->data += ONE_PRIVATE_REFERENCE; } - - ref->data.refcount++; return ref; } @@ -363,8 +330,8 @@ void SharedBufferRefExisting(PrivateRefCountEntry *ref) { Assert(ref != NULL); - Assert(ref->data.refcount > 0); - ref->data.refcount++; + Assert(!PrivateRefCountIsZero(ref->data)); + ref->data += ONE_PRIVATE_REFERENCE; } /* @@ -376,14 +343,14 @@ bool SharedBufferUnref(PrivateRefCountEntry *ref) { Assert(ref != NULL); - Assert(ref->data.refcount > 0); + Assert(!PrivateRefCountIsZero(ref->data)); - ref->data.refcount--; + ref->data -= ONE_PRIVATE_REFERENCE; - if (ref->data.refcount == 0) + if (PrivateRefCountIsZero(ref->data)) { /* No more references - clean up the entry */ - Assert(ref->data.lockmode == BUFFER_LOCK_UNLOCK); + Assert(SharedBufferGetLockMode(ref) == BUFFER_LOCK_UNLOCK); if (ref >= &PrivateRefCountArray[0] && ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES]) @@ -394,7 +361,8 @@ SharedBufferUnref(PrivateRefCountEntry *ref) } else { - refcount_delete_item(PrivateRefCountHash, ref); + /* could make slightly more efficient by using the pointer */ + refcount_delete(PrivateRefCountHash, ref->buffer); Assert(PrivateRefCountOverflowed > 0); PrivateRefCountOverflowed--; } @@ -406,24 +374,24 @@ SharedBufferUnref(PrivateRefCountEntry *ref) } /* - * Accessors for refcount entry fields. + * Accessors for refcount entry fields - opaque to callers. */ int32 SharedBufferRefCount(PrivateRefCountEntry *ref) { - return ref->data.refcount; + return PrivateRefCountGetRefCount(ref->data); } BufferLockMode SharedBufferGetLockMode(PrivateRefCountEntry *ref) { - return ref->data.lockmode; + return PrivateRefCountGetLockMode(ref->data); } void SharedBufferSetLockMode(PrivateRefCountEntry *ref, BufferLockMode mode) { - ref->data.lockmode = mode; + PrivateRefCountSetLockMode(ref->data, mode); } Buffer @@ -488,7 +456,7 @@ InitPrivateRefCountIterator(void) iter->array_index = 0; iter->in_hash = false; - iter->hash_iter_valid = false; + iter->hash_status = NULL; return iter; } @@ -514,20 +482,23 @@ GetNextPrivateRefCountEntry(PrivateRefCountIterator *iter) iter->in_hash = true; if (PrivateRefCountOverflowed > 0) { - refcount_start_iterate(PrivateRefCountHash, &iter->hash_iter); - iter->hash_iter_valid = true; + refcount_iterator *hiter = palloc(sizeof(refcount_iterator)); + + refcount_start_iterate(PrivateRefCountHash, hiter); + iter->hash_status = hiter; } } - if (iter->hash_iter_valid) + if (iter->hash_status != NULL) { PrivateRefCountEntry *res; - res = refcount_iterate(PrivateRefCountHash, &iter->hash_iter); + res = refcount_iterate(PrivateRefCountHash, (refcount_iterator *) iter->hash_status); if (res != NULL) return res; - iter->hash_iter_valid = false; + pfree(iter->hash_status); + iter->hash_status = NULL; } return NULL; @@ -539,6 +510,8 @@ GetNextPrivateRefCountEntry(PrivateRefCountIterator *iter) void FreePrivateRefCountIterator(PrivateRefCountIterator *iter) { + if (iter->hash_status != NULL) + pfree(iter->hash_status); pfree(iter); } -- 2.53.0