From 30f913882b547424d53bf1d9fc69203badd57211 Mon Sep 17 00:00:00 2001 From: Hannu Krosing Date: Sun, 23 Aug 2026 20:08:55 +0000 Subject: [PATCH v2 2/8] Add Direct TOAST catalog, GUC, and reloptions infrastructure Introduce base structures, macros, and configuration options for Direct TOAST: - Add struct varatt_direct and VARTAG_DIRECT (tag 19) in varatt.h. - Add toast_flavour reloption ('plain' / 'direct') in reloptions.c. - Add default_toast_flavour GUC in guc_tables.c. - Update create_toast_table() to support chunk_tids and chunk_tid_offsets columns for direct TOAST tables, with partial B-Tree indexing on chunk_id. --- src/backend/access/common/reloptions.c | 20 +++++++++++ src/backend/catalog/toasting.c | 26 ++++++++++++-- src/backend/utils/misc/guc_tables.c | 7 ++++ src/include/access/toast_internals.h | 9 +++++ src/include/utils/rel.h | 8 +++++ src/include/varatt.h | 48 +++++++++++++++++++++++++- 6 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index c3f6f45f460..fa3056a839e 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -530,6 +530,13 @@ static relopt_enum_elt_def StdRdOptIndexCleanupValues[] = {(const char *) NULL} /* list terminator */ }; +static relopt_enum_elt_def toastFlavourOptValues[] = +{ + {"plain", TOAST_FLAVOUR_PLAIN}, + {"direct", TOAST_FLAVOUR_DIRECT}, + {(const char *) NULL} /* list terminator */ +}; + /* values from GistOptBufferingMode */ static relopt_enum_elt_def gistBufferingOptValues[] = { @@ -561,6 +568,17 @@ static relopt_enum enumRelOpts[] = STDRD_OPTION_VACUUM_INDEX_CLEANUP_NOT_SET, gettext_noop("Valid values are \"on\", \"off\", and \"auto\".") }, + { + { + "toast_flavour", + "Sets the TOAST flavour for this table", + RELOPT_KIND_HEAP, + ShareUpdateExclusiveLock + }, + toastFlavourOptValues, + TOAST_FLAVOUR_NOT_SET, + gettext_noop("Valid values are \"plain\" and \"direct\".") + }, { { "buffering", @@ -2092,6 +2110,8 @@ static const relopt_parse_elt stdRdOptionsTab[] = { offsetof(StdRdOptions, parallel_workers)}, {"vacuum_index_cleanup", RELOPT_TYPE_ENUM, offsetof(StdRdOptions, vacuum_index_cleanup)}, + {"toast_flavour", RELOPT_TYPE_ENUM, + offsetof(StdRdOptions, toast_flavour)}, {"vacuum_truncate", RELOPT_TYPE_TERNARY, offsetof(StdRdOptions, vacuum_truncate)}, {"vacuum_max_eager_freeze_failure_rate", RELOPT_TYPE_REAL, diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 4aa52a4bd25..5eaa41e7fc6 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -202,7 +202,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, "pg_toast_%u_index", relOid); /* this is pretty painful... need a tuple descriptor */ - tupdesc = CreateTemplateTupleDesc(3); + tupdesc = CreateTemplateTupleDesc(5); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "chunk_id", OIDOID, @@ -215,6 +215,14 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, "chunk_data", BYTEAOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, + "chunk_tids", + TIDARRAYOID, + -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, + "chunk_tid_offsets", + INT8ARRAYOID, + -1, 0); /* * Ensure that the toast table doesn't itself get toasted, or we'll be @@ -224,15 +232,21 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, TupleDescAttr(tupdesc, 0)->attstorage = TYPSTORAGE_PLAIN; TupleDescAttr(tupdesc, 1)->attstorage = TYPSTORAGE_PLAIN; TupleDescAttr(tupdesc, 2)->attstorage = TYPSTORAGE_PLAIN; + TupleDescAttr(tupdesc, 3)->attstorage = TYPSTORAGE_PLAIN; + TupleDescAttr(tupdesc, 4)->attstorage = TYPSTORAGE_PLAIN; /* Toast field should not be compressed */ TupleDescAttr(tupdesc, 0)->attcompression = InvalidCompressionMethod; TupleDescAttr(tupdesc, 1)->attcompression = InvalidCompressionMethod; TupleDescAttr(tupdesc, 2)->attcompression = InvalidCompressionMethod; + TupleDescAttr(tupdesc, 3)->attcompression = InvalidCompressionMethod; + TupleDescAttr(tupdesc, 4)->attcompression = InvalidCompressionMethod; populate_compact_attribute(tupdesc, 0); populate_compact_attribute(tupdesc, 1); populate_compact_attribute(tupdesc, 2); + populate_compact_attribute(tupdesc, 3); + populate_compact_attribute(tupdesc, 4); TupleDescFinalize(tupdesc); @@ -299,7 +313,15 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, indexInfo->ii_IndexAttrNumbers[1] = 2; indexInfo->ii_Expressions = NIL; indexInfo->ii_ExpressionsState = NIL; - indexInfo->ii_Predicate = NIL; + { + NullTest *ntest = makeNode(NullTest); + + ntest->arg = (Expr *) makeVar(1, 1, OIDOID, -1, InvalidOid, 0); + ntest->nulltesttype = IS_NOT_NULL; + ntest->argisrow = false; + ntest->location = -1; + indexInfo->ii_Predicate = list_make1(ntest); + } indexInfo->ii_PredicateState = NULL; indexInfo->ii_ExclusionOps = NULL; indexInfo->ii_ExclusionProcs = NULL; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index c6d9b2a6f89..c600b3a2bd2 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -33,6 +33,7 @@ #include "access/gin.h" #include "access/slru.h" #include "access/toast_compression.h" +#include "access/toast_internals.h" #include "access/twophase.h" #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" @@ -287,6 +288,12 @@ static const struct config_enum_entry xmloption_options[] = { StaticAssertDecl(lengthof(xmloption_options) == (XMLOPTION_CONTENT + 2), "array length mismatch"); +static const struct config_enum_entry toast_flavour_options[] = { + {"plain", TOAST_FLAVOUR_PLAIN, false}, + {"direct", TOAST_FLAVOUR_DIRECT, false}, + {NULL, 0, false} +}; + /* * Although only "on", "off", and "safe_encoding" are documented, we * accept all the likely variants of "on" and "off". diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index bf45889a642..61836903c53 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -16,6 +16,15 @@ #include "storage/lockdefs.h" #include "utils/relcache.h" #include "utils/snapshot.h" +#include "utils/rel.h" + +#define RelationGetToastFlavour(relation) \ + ((relation)->rd_options ? \ + (((StdRdOptions *) (relation)->rd_options)->toast_flavour != TOAST_FLAVOUR_NOT_SET ? \ + ((StdRdOptions *) (relation)->rd_options)->toast_flavour : (ToastFlavour) toast_flavour) \ + : (ToastFlavour) toast_flavour) + +extern PGDLLIMPORT int toast_flavour; /* * The information at the start of the compressed toast data. diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 41ab4586c6b..034eae72887 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -341,6 +341,13 @@ typedef enum StdRdOptIndexCleanup STDRD_OPTION_VACUUM_INDEX_CLEANUP_NOT_SET, } StdRdOptIndexCleanup; +typedef enum ToastFlavour +{ + TOAST_FLAVOUR_NOT_SET = -1, + TOAST_FLAVOUR_PLAIN = 0, + TOAST_FLAVOUR_DIRECT = 1 +} ToastFlavour; + typedef struct StdRdOptions { int32 vl_len_; /* varlena header (do not touch directly!) */ @@ -357,6 +364,7 @@ typedef struct StdRdOptions * to freeze. 0 if disabled, -1 if unspecified. */ double vacuum_max_eager_freeze_failure_rate; + ToastFlavour toast_flavour; } StdRdOptions; #define HEAP_MIN_FILLFACTOR 10 diff --git a/src/include/varatt.h b/src/include/varatt.h index 000bdf33b92..7041e47754a 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -15,6 +15,8 @@ #ifndef VARATT_H #define VARATT_H +#include "storage/itemptr.h" + /* * varatt_external is a traditional "TOAST pointer", that is, the * information needed to fetch a Datum stored out-of-line in a TOAST table. @@ -38,6 +40,15 @@ typedef struct varatt_external Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external; +typedef struct varatt_direct +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ + ItemPointerData va_tid; /* Physical TID of the final chunk */ +} varatt_direct; + /* * These macros define the "saved size" portion of va_extinfo. Its remaining * two high-order bits identify the compression method. @@ -86,7 +97,8 @@ typedef enum vartag_external VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, - VARTAG_ONDISK = 18 + VARTAG_ONDISK = 18, + VARTAG_DIRECT = 19 } vartag_external; /* Is a TOAST pointer either type of expanded-object pointer? */ @@ -107,6 +119,8 @@ VARTAG_SIZE(vartag_external tag) return sizeof(varatt_expanded); else if (tag == VARTAG_ONDISK) return sizeof(varatt_external); + else if (tag == VARTAG_DIRECT) + return sizeof(varatt_direct); else { Assert(false); @@ -363,6 +377,19 @@ VARATT_IS_EXTERNAL_ONDISK(const void *PTR) return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK; } +static inline bool +VARATT_IS_EXTERNAL_DIRECT(const void *PTR) +{ + return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_DIRECT; +} + +static inline bool +VARATT_IS_EXTERNAL_ONDISK_OR_DIRECT(const void *PTR) +{ + return VARATT_IS_EXTERNAL(PTR) && + (VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK || VARTAG_EXTERNAL(PTR) == VARTAG_DIRECT); +} + /* Is varlena datum an indirect pointer? */ static inline bool VARATT_IS_EXTERNAL_INDIRECT(const void *PTR) @@ -539,4 +566,23 @@ VARATT_EXTERNAL_IS_COMPRESSED(varatt_external toast_pointer) (Size) (toast_pointer.va_rawsize - VARHDRSZ); } +static inline Size +VARATT_DIRECT_GET_EXTSIZE(varatt_direct toast_pointer) +{ + return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; +} + +static inline uint32 +VARATT_DIRECT_GET_COMPRESS_METHOD(varatt_direct toast_pointer) +{ + return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; +} + +static inline bool +VARATT_DIRECT_IS_COMPRESSED(varatt_direct toast_pointer) +{ + return VARATT_DIRECT_GET_EXTSIZE(toast_pointer) < + (Size) (toast_pointer.va_rawsize - VARHDRSZ); +} + #endif -- 2.55.0.970.g62bdec98f9-goog