From dc6172c6945354634063b03215dc6798ae22cc2f Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Veldanda Date: Sat, 3 May 2025 02:14:02 +0000 Subject: [PATCH v20 2/2] zstd nodict compression --- contrib/amcheck/verify_heapam.c | 1 + src/backend/access/common/detoast.c | 12 +- src/backend/access/common/reloptions.c | 14 +- src/backend/access/common/toast_compression.c | 171 +++++++++++++++++- src/backend/access/common/toast_internals.c | 4 + src/backend/utils/adt/varlena.c | 3 + src/backend/utils/misc/guc_tables.c | 3 + src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/psql/describe.c | 5 +- src/bin/psql/tab-complete.in.c | 4 +- src/include/access/toast_compression.h | 36 +++- src/include/access/toast_internals.h | 3 +- src/include/utils/attoptcache.h | 1 + src/test/regress/expected/compression.out | 5 +- src/test/regress/expected/compression_1.out | 3 + .../expected/compression_zstd_nodict.out | 152 ++++++++++++++++ .../expected/compression_zstd_nodict_1.out | 103 +++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/compression.sql | 1 + .../regress/sql/compression_zstd_nodict.sql | 82 +++++++++ 20 files changed, 581 insertions(+), 26 deletions(-) create mode 100644 src/test/regress/expected/compression_zstd_nodict.out create mode 100644 src/test/regress/expected/compression_zstd_nodict_1.out create mode 100644 src/test/regress/sql/compression_zstd_nodict.sql diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index d7c2ac6951a..111bb308341 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1792,6 +1792,7 @@ check_tuple_attribute(HeapCheckContext *ctx) /* List of all valid compression method IDs */ case TOAST_PGLZ_COMPRESSION_ID: case TOAST_LZ4_COMPRESSION_ID: + case TOAST_ZSTD_NODICT_COMPRESSION_ID: valid = true; break; diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index 01419d1c65f..451230023ec 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr, * Determine maximum amount of compressed data needed for a prefix * of a given length (after decompression). * - * At least for now, if it's LZ4 data, we'll have to fetch the - * whole thing, because there doesn't seem to be an API call to - * determine how much compressed data we need to be sure of being - * able to decompress the required slice. + * At least for now, if it's LZ4 or Zstandard data, we'll have to + * fetch the whole thing, because there doesn't seem to be an API + * call to determine how much compressed data we need to be sure + * of being able to decompress the required slice. */ if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) == TOAST_PGLZ_COMPRESSION_ID) @@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr) return pglz_decompress_datum(attr); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum(attr); + case TOAST_ZSTD_NODICT_COMPRESSION_ID: + return zstd_nodict_decompress_datum(attr); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ @@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength) return pglz_decompress_datum_slice(attr, slicelength); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum_slice(attr, slicelength); + case TOAST_ZSTD_NODICT_COMPRESSION_ID: + return zstd_nodict_decompress_datum_slice(attr, slicelength); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 46c1dce222d..1267668a242 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -24,6 +24,7 @@ #include "access/nbtree.h" #include "access/reloptions.h" #include "access/spgist_private.h" +#include "access/toast_compression.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/tablespace.h" @@ -381,7 +382,15 @@ static relopt_int intRelOpts[] = }, -1, 0, 1024 }, - + { + { + "zstd_level", + "Set column's ZSTD compression level", + RELOPT_KIND_ATTRIBUTE, + ShareUpdateExclusiveLock + }, + DEFAULT_ZSTD_LEVEL, MIN_ZSTD_LEVEL, MAX_ZSTD_LEVEL + }, /* list terminator */ {{NULL}} }; @@ -2097,7 +2106,8 @@ attribute_reloptions(Datum reloptions, bool validate) { static const relopt_parse_elt tab[] = { {"n_distinct", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct)}, - {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)} + {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)}, + {"zstd_level", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_level)}, }; return (bytea *) build_reloptions(reloptions, validate, diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 5e5d42d80ef..02823d1c435 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -17,6 +17,10 @@ #include #endif +#ifdef USE_ZSTD +#include +#endif + #include "access/detoast.h" #include "access/toast_compression.h" #include "common/pg_lzcompress.h" @@ -26,11 +30,19 @@ /* GUC */ int default_toast_compression = TOAST_PGLZ_COMPRESSION; -#define NO_LZ4_SUPPORT() \ +#ifdef USE_ZSTD +#define ZSTD_CHECK_ERROR(zstd_ret, msg) \ + do { \ + if (ZSTD_isError(zstd_ret)) \ + ereport(ERROR, (errmsg("%s: %s", (msg), ZSTD_getErrorName(zstd_ret)))); \ + } while (0) +#endif + +#define COMPRESSION_METHOD_NOT_SUPPORTED(method) \ ereport(ERROR, \ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ - errmsg("compression method lz4 not supported"), \ - errdetail("This functionality requires the server to be built with lz4 support."))) + errmsg("compression method %s not supported", method), \ + errdetail("This functionality requires the server to be built with %s support.", method))) /* * Compress a varlena using PGLZ. @@ -140,7 +152,7 @@ struct varlena * lz4_compress_datum(const struct varlena *value) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + COMPRESSION_METHOD_NOT_SUPPORTED("lz4"); return NULL; /* keep compiler quiet */ #else int32 valsize; @@ -183,7 +195,7 @@ struct varlena * lz4_decompress_datum(const struct varlena *value) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + COMPRESSION_METHOD_NOT_SUPPORTED("lz4"); return NULL; /* keep compiler quiet */ #else int32 rawsize; @@ -216,7 +228,7 @@ struct varlena * lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + COMPRESSION_METHOD_NOT_SUPPORTED("lz4"); return NULL; /* keep compiler quiet */ #else int32 rawsize; @@ -246,6 +258,129 @@ lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength) #endif } +/* Compress datum using ZSTD with optional dictionary (using cdict) */ +struct varlena * +zstd_nodict_compress_datum(const struct varlena *value, CompressionInfo cmp) +{ +#ifdef USE_ZSTD + uint32 valsize = VARSIZE_ANY_EXHDR(value); + size_t max_size = ZSTD_compressBound(valsize); + struct varlena *compressed; + void *dest; + size_t cmp_size; + + /* Allocate space for the compressed varlena (header + data) */ + compressed = (struct varlena *) palloc(max_size + VARATT_4BCE_HDRSZ(cmp.cmp_ext)); + dest = (char *) compressed + VARATT_4BCE_HDRSZ(cmp.cmp_ext); + + cmp_size = ZSTD_compress(dest, + max_size, + VARDATA_ANY(value), + valsize, + cmp.zstd_level); + + if (ZSTD_isError(cmp_size)) + { + pfree(compressed); + ZSTD_CHECK_ERROR(cmp_size, "ZSTD compression failed"); + } + + /* + * If compression did not reduce size, return NULL so that the + * uncompressed data is stored + */ + if (cmp_size > valsize) + { + pfree(compressed); + return NULL; + } + + /* Set the compressed size in the varlena header */ + SET_VARSIZE_COMPRESSED(compressed, cmp_size + VARATT_4BCE_HDRSZ(cmp.cmp_ext)); + return compressed; + +#else + COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict"); + return NULL; +#endif +} + +/* Decompression routine */ +struct varlena * +zstd_nodict_decompress_datum(const struct varlena *value) +{ +#ifdef USE_ZSTD + uint32 actual_size_exhdr = VARDATA_COMPRESSED_GET_EXTSIZE(value); + uint32 cmp_size_exhdr = VARATT_4BCE_PAYLOAD_SIZE(value); + struct varlena *result; + size_t uncmp_size; + + /* Allocate space for the uncompressed data */ + result = (struct varlena *) palloc(actual_size_exhdr + VARHDRSZ); + + uncmp_size = ZSTD_decompress(VARDATA(result), + actual_size_exhdr, + VARATT_4BCE_PAYLOAD_PTR(value), + cmp_size_exhdr); + + if (ZSTD_isError(uncmp_size)) + { + pfree(result); + ZSTD_CHECK_ERROR(uncmp_size, "ZSTD decompression failed"); + } + + /* Set final size in the varlena header */ + SET_VARSIZE(result, uncmp_size + VARHDRSZ); + return result; + +#else + COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict"); + return NULL; +#endif +} + +/* Decompress a slice of the datum using the streaming API and optional dictionary */ +struct varlena * +zstd_nodict_decompress_datum_slice(const struct varlena *value, int32 slicelength) +{ +#ifdef USE_ZSTD + struct varlena *result; + ZSTD_inBuffer inBuf; + ZSTD_outBuffer outBuf; + size_t ret; + ZSTD_DCtx *ZstdDecompressionCtx = ZSTD_createDCtx(); + + inBuf.src = VARATT_4BCE_PAYLOAD_PTR(value); + inBuf.size = VARATT_4BCE_PAYLOAD_SIZE(value); + inBuf.pos = 0; + + result = (struct varlena *) palloc(slicelength + VARHDRSZ); + outBuf.dst = (char *) result + VARHDRSZ; + outBuf.size = slicelength; + outBuf.pos = 0; + + /* Common decompression loop */ + while (inBuf.pos < inBuf.size && outBuf.pos < outBuf.size) + { + ret = ZSTD_decompressStream(ZstdDecompressionCtx, &outBuf, &inBuf); + if (ZSTD_isError(ret)) + { + pfree(result); + ZSTD_freeDCtx(ZstdDecompressionCtx); + ZSTD_CHECK_ERROR(ret, "zstd decompression failed"); + } + } + + Assert(outBuf.size == slicelength && outBuf.pos == slicelength); + SET_VARSIZE(result, outBuf.pos + VARHDRSZ); + ZSTD_freeDCtx(ZstdDecompressionCtx); + return result; +#else + COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict"); + return NULL; +#endif +} + /* * Extract compression ID from a varlena. * @@ -293,10 +428,17 @@ CompressionNameToMethod(const char *compression) else if (strcmp(compression, "lz4") == 0) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + COMPRESSION_METHOD_NOT_SUPPORTED("lz4"); #endif return TOAST_LZ4_COMPRESSION; } + else if (strcmp(compression, "zstd_nodict") == 0) + { +#ifndef USE_ZSTD + COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict"); +#endif + return TOAST_ZSTD_NODICT_COMPRESSION; + } return InvalidCompressionMethod; } @@ -313,6 +455,8 @@ GetCompressionMethodName(char method) return "pglz"; case TOAST_LZ4_COMPRESSION: return "lz4"; + case TOAST_ZSTD_NODICT_COMPRESSION: + return "zstd_nodict"; default: elog(ERROR, "invalid compression method %c", method); return NULL; /* keep compiler quiet */ @@ -326,6 +470,7 @@ setup_compression_info(char cmethod, Form_pg_attribute att) /* initialize from the attribute’s default settings */ info.cmethod = cmethod; + info.zstd_level = DEFAULT_ZSTD_LEVEL; info.cmp_ext = NULL; /* If the compression method is not valid, use the current default */ @@ -337,6 +482,18 @@ setup_compression_info(char cmethod, Form_pg_attribute att) case TOAST_PGLZ_COMPRESSION: case TOAST_LZ4_COMPRESSION: break; + case TOAST_ZSTD_NODICT_COMPRESSION: + { + AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum); + + if (aopt != NULL) + info.zstd_level = aopt->zstd_level; + + info.cmp_ext = palloc(sizeof(varatt_cmp_extended)); + + VARATT_4BCE_SET_HDR(info.cmp_ext, TOAST_ZSTD_NODICT_COMPRESSION_ID); + } + break; default: elog(ERROR, "invalid compression method %c", info.cmethod); } diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 83b537d51bf..5521d78bb48 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -68,6 +68,10 @@ toast_compress_datum(Datum value, CompressionInfo cmp) tmp = lz4_compress_datum((const struct varlena *) value); cmid = TOAST_LZ4_COMPRESSION_ID; break; + case TOAST_ZSTD_NODICT_COMPRESSION: + tmp = zstd_nodict_compress_datum((const struct varlena *) value, cmp); + cmid = TOAST_ZSTD_NODICT_COMPRESSION_ID; + break; default: elog(ERROR, "invalid compression method %c", cmp.cmethod); } diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 3e4d5568bde..5b9151c7e16 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -5301,6 +5301,9 @@ pg_column_compression(PG_FUNCTION_ARGS) case TOAST_LZ4_COMPRESSION_ID: result = "lz4"; break; + case TOAST_ZSTD_NODICT_COMPRESSION_ID: + result = "zstd_nodict"; + break; default: elog(ERROR, "invalid compression method id %d", cmid); } diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2f8cbd86759..948454e2093 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -460,6 +460,9 @@ static const struct config_enum_entry default_toast_compression_options[] = { {"pglz", TOAST_PGLZ_COMPRESSION, false}, #ifdef USE_LZ4 {"lz4", TOAST_LZ4_COMPRESSION, false}, +#endif +#ifdef USE_ZSTD + {"zstd_nodict", TOAST_ZSTD_NODICT_COMPRESSION, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 34826d01380..f2d2ca39514 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -756,7 +756,7 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate #row_security = on #default_table_access_method = 'heap' #default_tablespace = '' # a tablespace name, '' uses the default -#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' or 'zstd_nodict' #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace #check_function_bodies = on diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 1d08268393e..3831a7fab03 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2171,8 +2171,9 @@ describeOneTableDetails(const char *schemaname, /* these strings are literal in our syntax, so not translated. */ printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" : (compression[0] == 'l' ? "lz4" : - (compression[0] == '\0' ? "" : - "???"))), + (compression[0] == 'n' ? "zstd_nodict" : + (compression[0] == '\0' ? "" : + "???")))), false, false); } diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index c916b9299a8..2441acf41ce 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -2875,11 +2875,11 @@ match_previous_words(int pattern_id, /* ALTER TABLE ALTER [COLUMN] SET ( */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "(")) - COMPLETE_WITH("n_distinct", "n_distinct_inherited"); + COMPLETE_WITH("n_distinct", "n_distinct_inherited", "zstd_level"); /* ALTER TABLE ALTER [COLUMN] SET COMPRESSION */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION")) - COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4"); + COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4", "ZSTD_NODICT"); /* ALTER TABLE ALTER [COLUMN] SET EXPRESSION */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "EXPRESSION") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "EXPRESSION")) diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h index 1aef65cde99..94d56c7a4ef 100644 --- a/src/include/access/toast_compression.h +++ b/src/include/access/toast_compression.h @@ -16,6 +16,10 @@ #include "varatt.h" #include "catalog/pg_attribute.h" +#ifdef USE_ZSTD +#include +#endif + /* * GUC support. * @@ -36,6 +40,13 @@ unsupported_meta_size(const varatt_cmp_extended *hdr) return 0; /* unreachable */ } +/* no metadata for plain-ZSTD */ +static inline uint32 +zstd_nodict_meta_size(const varatt_cmp_extended *hdr) +{ + return 0; +} + /* * TOAST compression methods enumeration. * @@ -43,10 +54,11 @@ unsupported_meta_size(const varatt_cmp_extended *hdr) * VALUE : enum value * META-SIZE-FN : Calculates algorithm metadata size. */ -#define TOAST_COMPRESSION_LIST \ - X(PGLZ, 0, unsupported_meta_size) \ - X(LZ4, 1, unsupported_meta_size) \ - X(INVALID, 2, unsupported_meta_size) /* sentinel */ +#define TOAST_COMPRESSION_LIST \ + X(PGLZ, 0, unsupported_meta_size) \ + X(LZ4, 1, unsupported_meta_size) \ + X(ZSTD_NODICT, 2, zstd_nodict_meta_size) \ + X(INVALID, 3, unsupported_meta_size) /* sentinel */ /* Compression algorithm identifiers */ typedef enum ToastCompressionId @@ -96,6 +108,7 @@ toast_cmpid_meta_size(const varatt_cmp_extended *hdr) typedef struct CompressionInfo { char cmethod; + int zstd_level; varatt_cmp_extended *cmp_ext; /* non-NULL only if uses extended * compression methods */ } CompressionInfo; @@ -107,10 +120,20 @@ typedef struct CompressionInfo */ #define TOAST_PGLZ_COMPRESSION 'p' #define TOAST_LZ4_COMPRESSION 'l' +#define TOAST_ZSTD_NODICT_COMPRESSION 'n' #define InvalidCompressionMethod '\0' #define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod) +#ifdef USE_ZSTD +#define DEFAULT_ZSTD_LEVEL ZSTD_CLEVEL_DEFAULT +#define MIN_ZSTD_LEVEL (int)-ZSTD_BLOCKSIZE_MAX +#define MAX_ZSTD_LEVEL 22 +#else +#define DEFAULT_ZSTD_LEVEL 0 +#define MIN_ZSTD_LEVEL 0 +#define MAX_ZSTD_LEVEL 0 +#endif /* pglz compression/decompression routines */ extern struct varlena *pglz_compress_datum(const struct varlena *value); @@ -124,6 +147,11 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value); extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength); +/* zstd nodict compression/decompression routines */ +extern struct varlena *zstd_nodict_compress_datum(const struct varlena *value, CompressionInfo cmp); +extern struct varlena *zstd_nodict_decompress_datum(const struct varlena *value); +extern struct varlena *zstd_nodict_decompress_datum_slice(const struct varlena *value, int32 slicelength); + /* other stuff */ extern ToastCompressionId toast_get_compression_id(struct varlena *attr); extern char CompressionNameToMethod(const char *compression); diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index f4a4829ad17..b1859ef202b 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -35,7 +35,8 @@ typedef struct toast_compress_header do { \ Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \ Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm_method) == TOAST_LZ4_COMPRESSION_ID); \ + (cm_method) == TOAST_LZ4_COMPRESSION_ID || \ + (cm_method) == TOAST_ZSTD_NODICT_COMPRESSION_ID); \ if (!TOAST_CMPID_EXTENDED((cm_method))) \ { \ ((toast_compress_header *)(ptr))->tcinfo = \ diff --git a/src/include/utils/attoptcache.h b/src/include/utils/attoptcache.h index f684a772af5..51d65ebd646 100644 --- a/src/include/utils/attoptcache.h +++ b/src/include/utils/attoptcache.h @@ -21,6 +21,7 @@ typedef struct AttributeOpts int32 vl_len_; /* varlena header (do not touch directly!) */ float8 n_distinct; float8 n_distinct_inherited; + int zstd_level; } AttributeOpts; extern AttributeOpts *get_attribute_options(Oid attrelid, int attnum); diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out index 4dd9ee7200d..c7e108a0f52 100644 --- a/src/test/regress/expected/compression.out +++ b/src/test/regress/expected/compression.out @@ -238,10 +238,11 @@ NOTICE: merging multiple inherited definitions of column "f1" -- test default_toast_compression GUC SET default_toast_compression = ''; ERROR: invalid value for parameter "default_toast_compression": "" -HINT: Available values: pglz, lz4. +HINT: Available values: pglz, lz4, zstd_nodict. SET default_toast_compression = 'I do not exist compression'; ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression" -HINT: Available values: pglz, lz4. +HINT: Available values: pglz, lz4, zstd_nodict. +SET default_toast_compression = 'zstd_nodict'; SET default_toast_compression = 'lz4'; SET default_toast_compression = 'pglz'; -- test alter compression method diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out index 7bd7642b4b9..5b10d8c5259 100644 --- a/src/test/regress/expected/compression_1.out +++ b/src/test/regress/expected/compression_1.out @@ -233,6 +233,9 @@ HINT: Available values: pglz. SET default_toast_compression = 'I do not exist compression'; ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression" HINT: Available values: pglz. +SET default_toast_compression = 'zstd_nodict'; +ERROR: invalid value for parameter "default_toast_compression": "zstd_nodict" +HINT: Available values: pglz. SET default_toast_compression = 'lz4'; ERROR: invalid value for parameter "default_toast_compression": "lz4" HINT: Available values: pglz. diff --git a/src/test/regress/expected/compression_zstd_nodict.out b/src/test/regress/expected/compression_zstd_nodict.out new file mode 100644 index 00000000000..d80814e0492 --- /dev/null +++ b/src/test/regress/expected/compression_zstd_nodict.out @@ -0,0 +1,152 @@ +\set HIDE_TOAST_COMPRESSION false +-- Ensure stable results regardless of the installation's default. +SET default_toast_compression = 'pglz'; +---------------------------------------------------------------- +-- 1. Create Test Table with Zstd Compression +---------------------------------------------------------------- +DROP TABLE IF EXISTS cmdata_zstd_nodict CASCADE; +NOTICE: table "cmdata_zstd_nodict" does not exist, skipping +CREATE TABLE cmdata_zstd_nodict ( + f1 TEXT COMPRESSION zstd_nodict +); +---------------------------------------------------------------- +-- 2. Insert Data Rows +---------------------------------------------------------------- +DO $$ +BEGIN + FOR i IN 1..15 LOOP + INSERT INTO cmdata_zstd_nodict (f1) VALUES (repeat('1234567890', 1004)); + END LOOP; +END $$; +---------------------------------------------------------------- +-- 3. Verify Table Structure and Compression Settings +---------------------------------------------------------------- +-- Table Structure for cmdata_zstd +\d+ cmdata_zstd_nodict; + Table "public.cmdata_zstd_nodict" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd_nodict | | + +-- Compression Settings for f1 Column +SELECT pg_column_compression(f1) AS compression_method, + count(*) AS row_count +FROM cmdata_zstd_nodict +GROUP BY pg_column_compression(f1); + compression_method | row_count +--------------------+----------- + zstd_nodict | 15 +(1 row) + +---------------------------------------------------------------- +-- 4. Decompression Tests +---------------------------------------------------------------- +-- Decompression Slice Test (Extracting Substrings) +SELECT SUBSTR(f1, 200, 50) AS data_slice +FROM cmdata_zstd_nodict; + data_slice +---------------------------------------------------- + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 + 01234567890123456789012345678901234567890123456789 +(15 rows) + +---------------------------------------------------------------- +-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION +---------------------------------------------------------------- +DROP TABLE IF EXISTS cmdata_zstd_nodict_2; +NOTICE: table "cmdata_zstd_nodict_2" does not exist, skipping +CREATE TABLE cmdata_zstd_nodict_2 (LIKE cmdata_zstd_nodict INCLUDING COMPRESSION); +-- Table Structure for cmdata_zstd_2 +\d+ cmdata_zstd_nodict_2; + Table "public.cmdata_zstd_nodict_2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd_nodict | | + +DROP TABLE cmdata_zstd_nodict_2; +---------------------------------------------------------------- +-- 6. Materialized View Compression Test +---------------------------------------------------------------- +DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd_nodict; +NOTICE: materialized view "compressmv_zstd_nodict" does not exist, skipping +CREATE MATERIALIZED VIEW compressmv_zstd_nodict AS + SELECT f1 FROM cmdata_zstd_nodict; +-- Materialized View Structure for compressmv_zstd +\d+ compressmv_zstd_nodict; + Materialized view "public.compressmv_zstd_nodict" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | | | +View definition: + SELECT f1 + FROM cmdata_zstd_nodict; + +-- Materialized View Compression Check +SELECT pg_column_compression(f1) AS mv_compression +FROM compressmv_zstd_nodict; + mv_compression +---------------- + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict + zstd_nodict +(15 rows) + +---------------------------------------------------------------- +-- 7. Additional Updates and Round-Trip Tests +---------------------------------------------------------------- +-- Update some rows to check if the dictionary remains effective after modifications. +UPDATE cmdata_zstd_nodict +SET f1 = f1 || ' UPDATED'; +-- Verification of Updated Rows +SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview +FROM cmdata_zstd_nodict; + preview +--------- + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED + UPDATED +(15 rows) + +---------------------------------------------------------------- +-- 8. Clean Up +---------------------------------------------------------------- +DROP MATERIALIZED VIEW compressmv_zstd_nodict; +DROP TABLE cmdata_zstd_nodict; +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/expected/compression_zstd_nodict_1.out b/src/test/regress/expected/compression_zstd_nodict_1.out new file mode 100644 index 00000000000..161d11fcae2 --- /dev/null +++ b/src/test/regress/expected/compression_zstd_nodict_1.out @@ -0,0 +1,103 @@ +\set HIDE_TOAST_COMPRESSION false +-- Ensure stable results regardless of the installation's default. +SET default_toast_compression = 'pglz'; +---------------------------------------------------------------- +-- 1. Create Test Table with Zstd Compression +---------------------------------------------------------------- +DROP TABLE IF EXISTS cmdata_zstd_nodict CASCADE; +NOTICE: table "cmdata_zstd_nodict" does not exist, skipping +CREATE TABLE cmdata_zstd_nodict ( + f1 TEXT COMPRESSION zstd_nodict +); +ERROR: compression method zstd_nodict not supported +DETAIL: This functionality requires the server to be built with zstd_nodict support. +---------------------------------------------------------------- +-- 2. Insert Data Rows +---------------------------------------------------------------- +DO $$ +BEGIN + FOR i IN 1..15 LOOP + INSERT INTO cmdata_zstd_nodict (f1) VALUES (repeat('1234567890', 1004)); + END LOOP; +END $$; +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 1: INSERT INTO cmdata_zstd_nodict (f1) VALUES (repeat('12345678... + ^ +QUERY: INSERT INTO cmdata_zstd_nodict (f1) VALUES (repeat('1234567890', 1004)) +CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement +---------------------------------------------------------------- +-- 3. Verify Table Structure and Compression Settings +---------------------------------------------------------------- +-- Table Structure for cmdata_zstd +\d+ cmdata_zstd_nodict; +-- Compression Settings for f1 Column +SELECT pg_column_compression(f1) AS compression_method, + count(*) AS row_count +FROM cmdata_zstd_nodict +GROUP BY pg_column_compression(f1); +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 3: FROM cmdata_zstd_nodict + ^ +---------------------------------------------------------------- +-- 4. Decompression Tests +---------------------------------------------------------------- +-- Decompression Slice Test (Extracting Substrings) +SELECT SUBSTR(f1, 200, 50) AS data_slice +FROM cmdata_zstd_nodict; +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 2: FROM cmdata_zstd_nodict; + ^ +---------------------------------------------------------------- +-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION +---------------------------------------------------------------- +DROP TABLE IF EXISTS cmdata_zstd_nodict_2; +NOTICE: table "cmdata_zstd_nodict_2" does not exist, skipping +CREATE TABLE cmdata_zstd_nodict_2 (LIKE cmdata_zstd_nodict INCLUDING COMPRESSION); +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 1: CREATE TABLE cmdata_zstd_nodict_2 (LIKE cmdata_zstd_nodict I... + ^ +-- Table Structure for cmdata_zstd_2 +\d+ cmdata_zstd_nodict_2; +DROP TABLE cmdata_zstd_nodict_2; +ERROR: table "cmdata_zstd_nodict_2" does not exist +---------------------------------------------------------------- +-- 6. Materialized View Compression Test +---------------------------------------------------------------- +DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd_nodict; +NOTICE: materialized view "compressmv_zstd_nodict" does not exist, skipping +CREATE MATERIALIZED VIEW compressmv_zstd_nodict AS + SELECT f1 FROM cmdata_zstd_nodict; +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 2: SELECT f1 FROM cmdata_zstd_nodict; + ^ +-- Materialized View Structure for compressmv_zstd +\d+ compressmv_zstd_nodict; +-- Materialized View Compression Check +SELECT pg_column_compression(f1) AS mv_compression +FROM compressmv_zstd_nodict; +ERROR: relation "compressmv_zstd_nodict" does not exist +LINE 2: FROM compressmv_zstd_nodict; + ^ +---------------------------------------------------------------- +-- 7. Additional Updates and Round-Trip Tests +---------------------------------------------------------------- +-- Update some rows to check if the dictionary remains effective after modifications. +UPDATE cmdata_zstd_nodict +SET f1 = f1 || ' UPDATED'; +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 1: UPDATE cmdata_zstd_nodict + ^ +-- Verification of Updated Rows +SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview +FROM cmdata_zstd_nodict; +ERROR: relation "cmdata_zstd_nodict" does not exist +LINE 2: FROM cmdata_zstd_nodict; + ^ +---------------------------------------------------------------- +-- 8. Clean Up +---------------------------------------------------------------- +DROP MATERIALIZED VIEW compressmv_zstd_nodict; +ERROR: materialized view "compressmv_zstd_nodict" does not exist +DROP TABLE cmdata_zstd_nodict; +ERROR: table "cmdata_zstd_nodict" does not exist +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index a424be2a6bf..75cea22c418 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -123,7 +123,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr # The stats test resets stats, so nothing else needing stats access can be in # this group. # ---------- -test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression memoize stats predicate numa +test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression compression_zstd_nodict memoize stats predicate numa # event_trigger depends on create_am and cannot run concurrently with # any test that runs DDL diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql index 490595fcfb2..27979eb7997 100644 --- a/src/test/regress/sql/compression.sql +++ b/src/test/regress/sql/compression.sql @@ -102,6 +102,7 @@ CREATE TABLE cminh() INHERITS (cmdata, cmdata3); -- test default_toast_compression GUC SET default_toast_compression = ''; SET default_toast_compression = 'I do not exist compression'; +SET default_toast_compression = 'zstd_nodict'; SET default_toast_compression = 'lz4'; SET default_toast_compression = 'pglz'; diff --git a/src/test/regress/sql/compression_zstd_nodict.sql b/src/test/regress/sql/compression_zstd_nodict.sql new file mode 100644 index 00000000000..e1f18004cc9 --- /dev/null +++ b/src/test/regress/sql/compression_zstd_nodict.sql @@ -0,0 +1,82 @@ +\set HIDE_TOAST_COMPRESSION false + +-- Ensure stable results regardless of the installation's default. +SET default_toast_compression = 'pglz'; + +---------------------------------------------------------------- +-- 1. Create Test Table with Zstd Compression +---------------------------------------------------------------- +DROP TABLE IF EXISTS cmdata_zstd_nodict CASCADE; +CREATE TABLE cmdata_zstd_nodict ( + f1 TEXT COMPRESSION zstd_nodict +); + +---------------------------------------------------------------- +-- 2. Insert Data Rows +---------------------------------------------------------------- +DO $$ +BEGIN + FOR i IN 1..15 LOOP + INSERT INTO cmdata_zstd_nodict (f1) VALUES (repeat('1234567890', 1004)); + END LOOP; +END $$; + +---------------------------------------------------------------- +-- 3. Verify Table Structure and Compression Settings +---------------------------------------------------------------- +-- Table Structure for cmdata_zstd +\d+ cmdata_zstd_nodict; + +-- Compression Settings for f1 Column +SELECT pg_column_compression(f1) AS compression_method, + count(*) AS row_count +FROM cmdata_zstd_nodict +GROUP BY pg_column_compression(f1); + +---------------------------------------------------------------- +-- 4. Decompression Tests +---------------------------------------------------------------- +-- Decompression Slice Test (Extracting Substrings) +SELECT SUBSTR(f1, 200, 50) AS data_slice +FROM cmdata_zstd_nodict; + +---------------------------------------------------------------- +-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION +---------------------------------------------------------------- +DROP TABLE IF EXISTS cmdata_zstd_nodict_2; +CREATE TABLE cmdata_zstd_nodict_2 (LIKE cmdata_zstd_nodict INCLUDING COMPRESSION); +-- Table Structure for cmdata_zstd_2 +\d+ cmdata_zstd_nodict_2; +DROP TABLE cmdata_zstd_nodict_2; + +---------------------------------------------------------------- +-- 6. Materialized View Compression Test +---------------------------------------------------------------- +DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd_nodict; +CREATE MATERIALIZED VIEW compressmv_zstd_nodict AS + SELECT f1 FROM cmdata_zstd_nodict; + +-- Materialized View Structure for compressmv_zstd +\d+ compressmv_zstd_nodict; + +-- Materialized View Compression Check +SELECT pg_column_compression(f1) AS mv_compression +FROM compressmv_zstd_nodict; + +---------------------------------------------------------------- +-- 7. Additional Updates and Round-Trip Tests +---------------------------------------------------------------- +-- Update some rows to check if the dictionary remains effective after modifications. +UPDATE cmdata_zstd_nodict +SET f1 = f1 || ' UPDATED'; + +-- Verification of Updated Rows +SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview +FROM cmdata_zstd_nodict; +---------------------------------------------------------------- +-- 8. Clean Up +---------------------------------------------------------------- +DROP MATERIALIZED VIEW compressmv_zstd_nodict; +DROP TABLE cmdata_zstd_nodict; + +\set HIDE_TOAST_COMPRESSION true -- 2.47.1