From bd5f275e0afffa4909ce5839f9033abed448684c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 13 Aug 2025 17:26:36 +0900 Subject: [PATCH v5 02/15] Refactor some TOAST value ID code to use Oid8 instead of Oid This change is a mechanical switch to change most of the code paths that assume TOAST value IDs to be Oids to become Oid8, easing an upcoming change to allow larger TOAST values, at 8 bytes. The areas touched are related to table AM, amcheck and logical decoding's reorder buffer. A good chunk of the changes involve switching printf() markers from %u to OID8_FORMAT. --- src/include/access/heaptoast.h | 2 +- src/include/access/tableam.h | 4 +- src/backend/access/common/toast_internals.c | 8 +-- src/backend/access/heap/heaptoast.c | 12 ++-- .../replication/logical/reorderbuffer.c | 14 +++-- contrib/amcheck/verify_heapam.c | 56 +++++++++++-------- 6 files changed, 53 insertions(+), 43 deletions(-) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 6385a27caf83..fdc8d00d7099 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -142,7 +142,7 @@ extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc, * Fetch a slice from a toast value stored in a heap table. * ---------- */ -extern void heap_fetch_toast_slice(Relation toastrel, Oid valueid, +extern void heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result); diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 1c9e802a6b12..0164083ddc75 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -740,7 +740,7 @@ typedef struct TableAmRoutine * table implemented by this AM. See table_relation_fetch_toast_slice() * for more details. */ - void (*relation_fetch_toast_slice) (Relation toastrel, Oid valueid, + void (*relation_fetch_toast_slice) (Relation toastrel, Oid8 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, @@ -1873,7 +1873,7 @@ table_relation_toast_am(Relation rel) * stored. */ static inline void -table_relation_fetch_toast_slice(Relation toastrel, Oid valueid, +table_relation_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result) { diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index a1d0eed8953b..8d8f12a0c256 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -26,8 +26,8 @@ #include "utils/rel.h" #include "utils/snapmgr.h" -static bool toastrel_valueid_exists(Relation toastrel, Oid valueid); -static bool toastid_valueid_exists(Oid toastrelid, Oid valueid); +static bool toastrel_valueid_exists(Relation toastrel, Oid8 valueid); +static bool toastid_valueid_exists(Oid toastrelid, Oid8 valueid); /* ---------- * toast_compress_datum - @@ -456,7 +456,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) * ---------- */ static bool -toastrel_valueid_exists(Relation toastrel, Oid valueid) +toastrel_valueid_exists(Relation toastrel, Oid8 valueid) { bool result = false; ScanKeyData toastkey; @@ -504,7 +504,7 @@ toastrel_valueid_exists(Relation toastrel, Oid valueid) * ---------- */ static bool -toastid_valueid_exists(Oid toastrelid, Oid valueid) +toastid_valueid_exists(Oid toastrelid, Oid8 valueid) { bool result; Relation toastrel; diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index cb1e57030f64..d4b600de3aca 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -623,7 +623,7 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, * result is the varlena into which the results should be written. */ void -heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, +heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result) { @@ -725,7 +725,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, else { /* should never happen */ - elog(ERROR, "found toasted toast chunk for toast value %u in %s", + elog(ERROR, "found toasted toast chunk for toast value " OID8_FORMAT " in %s", valueid, RelationGetRelationName(toastrel)); chunksize = 0; /* keep compiler quiet */ chunkdata = NULL; @@ -737,13 +737,13 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, if (curchunk != expectedchunk) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk number %d (expected %d) for toast value %u in %s", + errmsg_internal("unexpected chunk number %d (expected %d) for toast value " OID8_FORMAT " in %s", curchunk, expectedchunk, valueid, RelationGetRelationName(toastrel)))); if (curchunk > endchunk) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast value %u in %s", + errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast value " OID8_FORMAT " in %s", curchunk, startchunk, endchunk, valueid, RelationGetRelationName(toastrel)))); @@ -752,7 +752,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, if (chunksize != expected_size) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s", + errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for toast value " OID8_FORMAT " in %s", chunksize, expected_size, curchunk, totalchunks, valueid, RelationGetRelationName(toastrel)))); @@ -781,7 +781,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, if (expectedchunk != (endchunk + 1)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("missing chunk number %d for toast value %u in %s", + errmsg_internal("missing chunk number %d for toast value " OID8_FORMAT " in %s", expectedchunk, valueid, RelationGetRelationName(toastrel)))); diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 34cf05668ae8..1c1c203b4145 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -176,7 +176,7 @@ typedef struct ReorderBufferIterTXNState /* toast datastructures */ typedef struct ReorderBufferToastEnt { - Oid chunk_id; /* toast_table.chunk_id */ + Oid8 chunk_id; /* toast_table.chunk_id */ int32 last_chunk_seq; /* toast_table.chunk_seq of the last chunk we * have seen */ Size num_chunks; /* number of chunks we've already seen */ @@ -4944,7 +4944,7 @@ ReorderBufferToastInitHash(ReorderBuffer *rb, ReorderBufferTXN *txn) Assert(txn->toast_hash == NULL); - hash_ctl.keysize = sizeof(Oid); + hash_ctl.keysize = sizeof(Oid8); hash_ctl.entrysize = sizeof(ReorderBufferToastEnt); hash_ctl.hcxt = rb->context; txn->toast_hash = hash_create("ReorderBufferToastHash", 5, &hash_ctl, @@ -4968,7 +4968,7 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, bool isnull; Pointer chunk; TupleDesc desc = RelationGetDescr(relation); - Oid chunk_id; + Oid8 chunk_id; int32 chunk_seq; if (txn->toast_hash == NULL) @@ -4995,11 +4995,11 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, dlist_init(&ent->chunks); if (chunk_seq != 0) - elog(ERROR, "got sequence entry %d for toast chunk %u instead of seq 0", + elog(ERROR, "got sequence entry %d for toast chunk " OID8_FORMAT " instead of seq 0", chunk_seq, chunk_id); } else if (found && chunk_seq != ent->last_chunk_seq + 1) - elog(ERROR, "got sequence entry %d for toast chunk %u instead of seq %d", + elog(ERROR, "got sequence entry %d for toast chunk " OID8_FORMAT " instead of seq %d", chunk_seq, chunk_id, ent->last_chunk_seq + 1); chunk = DatumGetPointer(fastgetattr(newtup, 3, desc, &isnull)); @@ -5108,6 +5108,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, struct varlena *reconstructed; dlist_iter it; Size data_done = 0; + Oid8 toast_valueid; /* system columns aren't toasted */ if (attr->attnum < 0) @@ -5132,13 +5133,14 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, continue; VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena); + toast_valueid = toast_pointer.va_valueid; /* * Check whether the toast tuple changed, replace if so. */ ent = (ReorderBufferToastEnt *) hash_search(txn->toast_hash, - &toast_pointer.va_valueid, + &toast_valueid, HASH_FIND, NULL); if (ent == NULL) diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 4963e9245cb5..eb353c40249e 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1561,6 +1561,9 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, bool isnull; int32 chunksize; int32 expected_size; + Oid8 toast_valueid; + + toast_valueid = ta->toast_pointer.va_valueid; /* Sanity-check the sequence number. */ chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2, @@ -1568,16 +1571,16 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (isnull) { report_toast_corruption(ctx, ta, - psprintf("toast value %u has toast chunk with null sequence number", - ta->toast_pointer.va_valueid)); + psprintf("toast value " OID8_FORMAT " has toast chunk with null sequence number", + toast_valueid)); return; } if (chunk_seq != *expected_chunk_seq) { /* Either the TOAST index is corrupt, or we don't have all chunks. */ report_toast_corruption(ctx, ta, - psprintf("toast value %u index scan returned chunk %d when expecting chunk %d", - ta->toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " index scan returned chunk %d when expecting chunk %d", + toast_valueid, chunk_seq, *expected_chunk_seq)); } *expected_chunk_seq = chunk_seq + 1; @@ -1588,8 +1591,8 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (isnull) { report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d has null data", - ta->toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " chunk %d has null data", + toast_valueid, chunk_seq)); return; } @@ -1608,8 +1611,8 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, uint32 header = ((varattrib_4b *) chunk)->va_4byte.va_header; report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d has invalid varlena header %0x", - ta->toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " chunk %d has invalid varlena header %0x", + toast_valueid, chunk_seq, header)); return; } @@ -1620,8 +1623,8 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (chunk_seq > last_chunk_seq) { report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d follows last expected chunk %d", - ta->toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " chunk %d follows last expected chunk %d", + toast_valueid, chunk_seq, last_chunk_seq)); return; } @@ -1631,8 +1634,8 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (chunksize != expected_size) report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d has size %u, but expected size %u", - ta->toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " chunk %d has size %u, but expected size %u", + toast_valueid, chunk_seq, chunksize, expected_size)); } @@ -1663,6 +1666,7 @@ check_tuple_attribute(HeapCheckContext *ctx) struct varlena *attr; char *tp; /* pointer to the tuple data */ uint16 infomask; + Oid8 toast_pointer_valueid; CompactAttribute *thisatt; struct varatt_external toast_pointer; @@ -1771,12 +1775,13 @@ check_tuple_attribute(HeapCheckContext *ctx) * Must copy attr into toast_pointer for alignment considerations */ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_pointer_valueid = toast_pointer.va_valueid; /* Toasted attributes too large to be untoasted should never be stored */ if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT) report_corruption(ctx, - psprintf("toast value %u rawsize %d exceeds limit %d", - toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d", + toast_pointer_valueid, toast_pointer.va_rawsize, VARLENA_SIZE_LIMIT)); @@ -1803,16 +1808,16 @@ check_tuple_attribute(HeapCheckContext *ctx) } if (!valid) report_corruption(ctx, - psprintf("toast value %u has invalid compression method id %d", - toast_pointer.va_valueid, cmid)); + psprintf("toast value " OID8_FORMAT " has invalid compression method id %d", + toast_pointer_valueid, cmid)); } /* The tuple header better claim to contain toasted values */ if (!(infomask & HEAP_HASEXTERNAL)) { report_corruption(ctx, - psprintf("toast value %u is external but tuple header flag HEAP_HASEXTERNAL not set", - toast_pointer.va_valueid)); + psprintf("toast value " OID8_FORMAT " is external but tuple header flag HEAP_HASEXTERNAL not set", + toast_pointer_valueid)); return true; } @@ -1820,8 +1825,8 @@ check_tuple_attribute(HeapCheckContext *ctx) if (!ctx->rel->rd_rel->reltoastrelid) { report_corruption(ctx, - psprintf("toast value %u is external but relation has no toast relation", - toast_pointer.va_valueid)); + psprintf("toast value " OID8_FORMAT " is external but relation has no toast relation", + toast_pointer_valueid)); return true; } @@ -1866,6 +1871,7 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) uint32 extsize; int32 expected_chunk_seq = 0; int32 last_chunk_seq; + Oid8 toast_valueid; extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer); last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE; @@ -1896,14 +1902,16 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) } systable_endscan_ordered(toastscan); + toast_valueid = ta->toast_pointer.va_valueid; + if (!found_toasttup) report_toast_corruption(ctx, ta, - psprintf("toast value %u not found in toast table", - ta->toast_pointer.va_valueid)); + psprintf("toast value " OID8_FORMAT " not found in toast table", + toast_valueid)); else if (expected_chunk_seq <= last_chunk_seq) report_toast_corruption(ctx, ta, - psprintf("toast value %u was expected to end at chunk %d, but ended while expecting chunk %d", - ta->toast_pointer.va_valueid, + psprintf("toast value " OID8_FORMAT " was expected to end at chunk %d, but ended while expecting chunk %d", + toast_valueid, last_chunk_seq, expected_chunk_seq)); } -- 2.50.0