From 4e5a25557aa9b6e2fc72d82e10ac1a6ede00e665 Mon Sep 17 00:00:00 2001 From: Hannu Krosing Date: Sun, 23 Aug 2026 20:17:07 +0000 Subject: [PATCH v12 06/10] Add documentation for Direct TOAST Update the PostgreSQL SGML documentation to describe Direct TOAST: - doc/src/sgml/storage.sgml: Document Plain and Direct on-disk TOAST formats, TID-based direct addressing, single chunk, flat arrays, and hierarchical DAGs. - doc/src/sgml/config.sgml: Document the toast_default_flavour configuration parameter. - doc/src/sgml/ref/create_table.sgml: Document the toast_flavour storage parameter. --- doc/src/sgml/config.sgml | 22 ++++ doc/src/sgml/ref/create_table.sgml | 23 +++++ doc/src/sgml/storage.sgml | 155 ++++++++++++++++++++++++++--- 3 files changed, 187 insertions(+), 13 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f36fbb60101..74e31c0c3e2 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -10509,6 +10509,28 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + toast_default_flavour (enum) + + toast_default_flavour configuration parameter + + + + + This variable sets the default TOAST table format + to use when creating new TOAST tables. Supported + values are plain (the default, which creates a + 3-column TOAST table with a non-partial primary key + index) and direct (which creates a 5-column Direct + TOAST table with a partial unique index). + Actual INSERT and UPDATE + operations use the table's toast_flavour storage + parameter in CREATE TABLE or + ALTER TABLE. + + + + temp_tablespaces (string) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index fef24d8f3a2..b46f32f27b8 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1683,6 +1683,29 @@ WITH ( MODULUS numeric_literal, REM + + toast_flavour (enum) + + toast_flavour storage parameter + + + + + Sets the TOAST storage flavour for out-of-line data + written to this table. Supported values are plain (the + default, which uses index-based chunk lookup) and direct + (which stores physical TID pointers to bypass index lookup). + When creating a new table, the initial TOAST table + format is also controlled by ; + setting toast_flavour = 'direct' via + ALTER TABLE on a table with a 3-column + TOAST table automatically upgrades its + TOAST table in-place to the 5-column Direct + TOAST format. + + + + parallel_workers (integer) diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 83de016eaa5..810e658f5a7 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -427,19 +427,148 @@ belonging to the owning table. Every chunk_id (an OID or an OID8 identifying the particular TOASTed value), chunk_seq (a sequence number for the chunk within its value), -and chunk_data (the actual data of the chunk). A unique index -on chunk_id and chunk_seq provides fast -retrieval of the values. A pointer datum representing an out-of-line on-disk -TOASTed value therefore needs to store the OID of the -TOAST table in which to look and the specific value -(its chunk_id). For convenience, pointer datums also store the -logical datum size (original uncompressed data length), physical stored size -(different if compression was applied), and the compression method used, if -any. Allowing for the varlena header bytes, -the total size of an on-disk TOAST pointer datum is 18 -bytes when using an OID as chunk_id, or 22 bytes -when using an OID8 as chunk_id, regardless of the -actual size of the represented value. +and chunk_data (the actual data of the chunk). +Direct TOAST tables (created when + is direct or the +owning table has toast_flavour = 'direct') additionally have +the columns chunk_tids (an array of chunk tuple identifiers) +and chunk_tid_offsets (an array of byte offsets, used for hierarchical Direct TOAST trees). + + + +PostgreSQL supports two flavours of on-disk TOAST +storage: + + + + Plain TOAST (the default): A unique index + on chunk_id and chunk_seq provides + retrieval of the values. A pointer datum representing a plain on-disk + TOASTed value stores the OID of the TOAST table + and the specific value (chunk_id). + Allowing for the varlena header bytes, the total size of a plain on-disk + TOAST pointer datum is 18 bytes when using an OID as + chunk_id, or 22 bytes when using an OID8 as + chunk_id, regardless of the actual size of the represented value. + + + + + Direct TOAST (toast_flavour = 'direct'): + Instead of looking up chunks through a B-Tree index, the TOAST + pointer directly stores the physical Tuple Identifier (TID) of the root/final chunk + in a 22-byte pointer datum. For single-chunk values (up to ~2 kB), reading the data + requires only a single buffer pin without any index access. For multi-chunk values up to 100 chunks, + the final chunk contains a chunk_tids array pointing directly + to all data chunks. For larger values (> 100 chunks), an internal hierarchical tree + is constructed with intermediate index nodes storing child TIDs and byte offsets in + chunk_tid_offsets, enabling efficient tree traversal and slice pruning. + In Direct TOAST, chunk_id is set to NULL, + and the unique index on the TOAST table is a partial index (WHERE chunk_id IS NOT NULL), + completely avoiding index maintenance and WAL logging overhead for direct chunks. + + + + + + +For convenience, pointer datums also store the logical datum size (original +uncompressed data length), physical stored size (different if compression was applied), +and the compression method used, if any. + + + +Direct TOAST is fully compatible with +pg_upgrade (including mode). +Because pg_upgrade --link preserves the physical block +and offset layout of heap and TOAST relation files, +physical TID pointers remain valid across major version upgrades. +Furthermore, plain and direct TOAST +pointers can seamlessly coexist within the same table, allowing tables upgraded from +older PostgreSQL versions to be read transparently and +gradually adopt Direct TOAST for new writes. +When a table has a 3-column TOAST table (either created with +toast_default_flavour set to plain or +upgraded from an older PostgreSQL version), executing +ALTER TABLE ... SET (toast_flavour = 'direct') automatically +upgrades the TOAST table in-place to the 5-column format with a +partial index. Administrators can also explicitly upgrade any table or TOAST +table by calling pg_ensure_direct_toast(regclass). + + + +To upgrade all user tables' TOAST tables in a quiet database in bulk, +the following query can be used: + +SELECT c.oid::regclass AS tablename, + pg_ensure_direct_toast(c.reltoastrelid::regclass) + FROM pg_class AS c + WHERE c.reltoastrelid != 0 + AND EXISTS (SELECT FROM pg_stat_user_tables AS t WHERE t.relid = c.oid); + + + + +For a busy production database, tables can be upgraded one by one with lock timeouts +and growing wait times between retries to avoid lock contention: + +DO $$ +DECLARE + lock_timeout_ms int := 100; + max_attempts int := 10; + tablename text; + toastoid regclass; + attempt_nr int; + update_completed boolean; + attempts text; + any_found boolean := false; +BEGIN + PERFORM set_config('lock_timeout', lock_timeout_ms || 'ms', false); + + FOR tablename, toastoid IN + SELECT c.oid::regclass::text, + c.reltoastrelid::regclass + FROM pg_class AS c + WHERE EXISTS (SELECT FROM pg_stat_user_tables AS t WHERE t.relid = c.oid) + AND (SELECT count(*) + FROM pg_attribute AS a + WHERE a.attrelid = c.reltoastrelid + AND a.attnum > 0) = 3 + LOOP + any_found := true; + update_completed := false; + FOR attempt_nr IN 1..max_attempts LOOP + BEGIN + PERFORM pg_ensure_direct_toast(toastoid); + update_completed := true; + EXIT; + EXCEPTION + WHEN lock_not_available THEN + PERFORM pg_sleep(0.1 * attempt_nr); /* sleep a little longer each time */ + WHEN OTHERS THEN + RAISE WARNING 'Error updating % to Direct TOAST: %', tablename, SQLERRM; + EXIT; + END; + END LOOP; + + IF update_completed THEN + IF attempt_nr > 1 THEN + attempts := format(' after %s attempts', attempt_nr); + ELSE + attempts := ''; + END IF; + RAISE INFO 'Table % updated to Direct TOAST%', tablename, attempts; + ELSE + RAISE WARNING 'Timeout waiting to update table % to Direct TOAST', tablename; + END IF; + END LOOP; + + IF NOT any_found THEN + RAISE INFO 'No tables needed update to use Direct TOAST'; + END IF; +END; +$$; + -- 2.56.0.rc1.315.gc6ed9934b7-goog