From cb3e89895a4335d9f1796272cf3465ea72784ab8 Mon Sep 17 00:00:00 2001 From: Shlok Kyal Date: Sun, 3 Aug 2025 21:01:12 +0530 Subject: [PATCH v20 3/3] Skip publishing the columns specified in FOR TABLE EXCEPT A new "TABLE table_name EXCEPT (column_list)" clause for CREATE/ALTER PUBLICATION allows one or more columns to be excluded. The publisher will not send the data of excluded columns to the subscriber. The new syntax allows specifying excluded column list when creating or altering a publication. For example: CREATE PUBLICATION pubname FOR TABLE tabname EXCEPT (col1, col2, col3) or ALTER PUBLICATION pubname ADD TABLE tabname EXCEPT (col1, col2, col3) When column "prexcept" of system catalog "pg_publication_rel" is set to "true", and column "prattrs" of system catalog "pg_publication_rel" is not NULL, that means the publication was created with "EXCEPT (column-list)", and the columns in "prattrs" will be excluded from being published. pg_dump is updated to identify and dump the excluded column list of the publication. The psql \d family of commands can now display excluded column list. e.g. psql \dRp+ variant will now display associated "EXCEPT (column_list)" if any. --- doc/src/sgml/catalogs.sgml | 5 +- doc/src/sgml/logical-replication.sgml | 106 ++++++-- doc/src/sgml/ref/alter_publication.sgml | 10 +- doc/src/sgml/ref/create_publication.sgml | 52 +++- src/backend/catalog/pg_publication.c | 75 +++++- src/backend/commands/publicationcmds.c | 54 ++-- src/backend/parser/gram.y | 44 ++-- src/backend/replication/logical/tablesync.c | 41 ++- src/backend/replication/pgoutput/pgoutput.c | 70 +++++- src/bin/pg_dump/pg_dump.c | 45 ++-- src/bin/pg_dump/pg_dump.h | 1 + src/bin/psql/describe.c | 237 ++++++++++-------- src/bin/psql/tab-complete.in.c | 4 +- src/include/catalog/pg_publication.h | 6 +- src/include/catalog/pg_publication_rel.h | 5 +- src/test/regress/expected/publication.out | 88 +++++++ src/test/regress/sql/publication.sql | 55 ++++ src/test/subscription/meson.build | 1 + .../t/037_rep_changes_except_collist.pl | 190 ++++++++++++++ 19 files changed, 875 insertions(+), 214 deletions(-) create mode 100644 src/test/subscription/t/037_rep_changes_except_collist.pl diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index e96a55fecf9..c14077caa68 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -6573,7 +6573,10 @@ SCRAM-SHA-256$<iteration count>:&l prexcept bool - True if the relation must be excluded + True if the column list or relation must be excluded from publication. + If a column list is specified in prattrs, then + exclude only those columns. If prattrs is null, + then exclude the entire relation. diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 414a314acc5..2f04f93620e 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -1372,10 +1372,10 @@ Publications: Column Lists - Each publication can optionally specify which columns of each table are - replicated to subscribers. The table on the subscriber side must have at - least all the columns that are published. If no column list is specified, - then all columns on the publisher are replicated. + Each publication can optionally specify which columns of each table should be + replicated or excluded from replication. The table on the subscriber side + must have at least all the columns that are published. If no column list is + specified, then all columns on the publisher are replicated. See for details on the syntax. @@ -1389,8 +1389,11 @@ Publications: If no column list is specified, any columns added to the table later are - automatically replicated. This means that having a column list which names - all columns is not the same as having no column list at all. + automatically replicated. However, a normal column list (without + EXCEPT) only replicates the specified columns and no more. + Therefore, having a column list that names all columns is not the same as + having no column list at all, as more columns may be added to the table + later. @@ -1402,6 +1405,14 @@ Publications: Generated columns can also be specified in a column list. This allows generated columns to be published, regardless of the publication parameter + publish_generated_columns. Specifying generated + columns using the EXCEPT clause excludes those columns + from being published, regardless of the + + publish_generated_columns setting. However, for + generated columns that are not listed in the EXCEPT + clause, whether they are published or not still depends on the value of + publish_generated_columns. See for details. @@ -1423,11 +1434,14 @@ Publications: If a publication publishes UPDATE or - DELETE operations, any column list must include the - table's replica identity columns (see - ). + DELETE operations, any column list must include table's + replica identity columns and any column list specified with + EXCEPT clause must not include the table's replica + identity columns (see ). If a publication publishes only INSERT operations, then - the column list may omit replica identity columns. + the column list may omit replica identity columns and the column list + specified with EXCEPT clause may include replica identity + columns. @@ -1472,18 +1486,21 @@ Publications: Examples - Create a table t1 to be used in the following example. + Create tables t1 and t2 to be used in + the following example. /* pub # */ CREATE TABLE t1(id int, a text, b text, c text, d text, e text, PRIMARY KEY(id)); +/* pub # */ CREATE TABLE t2(id int, a text, b text, c text, d text, e text, PRIMARY KEY(id)); Create a publication p1. A column list is defined for - table t1 to reduce the number of columns that will be - replicated. Notice that the order of column names in the column list does - not matter. + table t1, and another column list is defined for table + t2 using the EXCEPT clause to reduce + the number of columns that will be replicated. Note that the order of column + names in the column lists does not matter. -/* pub # */ CREATE PUBLICATION p1 FOR TABLE t1 (id, b, a, d); +/* pub # */ CREATE PUBLICATION p1 FOR TABLE t1 (id, b, a, d), t2 EXCEPT (d, a); @@ -1497,6 +1514,7 @@ Publications: postgres | f | t | t | t | t | none | f Tables: "public.t1" (id, a, b, d) + "public.t2" EXCEPT (a, d) @@ -1517,23 +1535,41 @@ Indexes: "t1_pkey" PRIMARY KEY, btree (id) Publications: "p1" (id, a, b, d) + +/* pub # */ \d t2 + Table "public.t2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + id | integer | | not null | + a | text | | | + b | text | | | + c | text | | | + d | text | | | + e | text | | | +Indexes: + "t2_pkey" PRIMARY KEY, btree (id) +Publications: + "p1" EXCEPT (a, d) - On the subscriber node, create a table t1 which now - only needs a subset of the columns that were on the publisher table - t1, and also create the subscription + On the subscriber node, create tables t1 and + t2 which now only needs a subset of the columns that + were on the publisher tables t1 and + t2, and also create the subscription s1 that subscribes to the publication p1. /* sub # */ CREATE TABLE t1(id int, b text, a text, d text, PRIMARY KEY(id)); +/* sub # */ CREATE TABLE t2(id int, b text, c text, e text, PRIMARY KEY(id)); /* sub # */ CREATE SUBSCRIPTION s1 /* sub - */ CONNECTION 'host=localhost dbname=test_pub application_name=s1' /* sub - */ PUBLICATION p1; - On the publisher node, insert some rows to table t1. + On the publisher node, insert some rows to tables t1 + and t2 /* pub # */ INSERT INTO t1 VALUES(1, 'a-1', 'b-1', 'c-1', 'd-1', 'e-1'); /* pub # */ INSERT INTO t1 VALUES(2, 'a-2', 'b-2', 'c-2', 'd-2', 'e-2'); @@ -1545,11 +1581,21 @@ Publications: 2 | a-2 | b-2 | c-2 | d-2 | e-2 3 | a-3 | b-3 | c-3 | d-3 | e-3 (3 rows) +/* pub # */ INSERT INTO t2 VALUES(1, 'a-1', 'b-1', 'c-1', 'd-1', 'e-1'); +/* pub # */ INSERT INTO t2 VALUES(2, 'a-2', 'b-2', 'c-2', 'd-2', 'e-2'); +/* pub # */ INSERT INTO t2 VALUES(3, 'a-3', 'b-3', 'c-3', 'd-3', 'e-3'); +/* pub # */ SELECT * FROM t2 ORDER BY id; + id | a | b | c | d | e +----+-----+-----+-----+-----+----- + 1 | a-1 | b-1 | c-1 | d-1 | e-1 + 2 | a-2 | b-2 | c-2 | d-2 | e-2 + 3 | a-3 | b-3 | c-3 | d-3 | e-3 +(3 rows) - Only data from the column list of publication p1 is - replicated. + Only data specified by the column lists of publication + p1 is replicated. /* sub # */ SELECT * FROM t1 ORDER BY id; id | b | a | d @@ -1558,6 +1604,13 @@ Publications: 2 | b-2 | a-2 | d-2 3 | b-3 | a-3 | d-3 (3 rows) +/* sub # */ SELECT * FROM t2 ORDER BY id; + id | b | c | e +----+-----+-----+----- + 1 | b-1 | c-1 | e-1 + 2 | b-2 | c-2 | e-2 + 3 | b-3 | c-3 | e-3 +(3 rows) @@ -1654,6 +1707,17 @@ Publications: + + Generated columns specified in the column list with the + EXCEPT clause are not published, regardless of the value + of the + publish_generated_columns parameter. However, + generated columns that are not part of the column list with the + EXCEPT clause are published according to the value of the + + publish_generated_columns parameter. + + The following table summarizes behavior when there are generated columns involved in the logical replication. Results are shown for when diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml index 38b4657378a..f79ef789d93 100644 --- a/doc/src/sgml/ref/alter_publication.sgml +++ b/doc/src/sgml/ref/alter_publication.sgml @@ -32,7 +32,7 @@ ALTER PUBLICATION name RESET where publication_object is one of: - TABLE [ ONLY ] table_name [ * ] [ ( column_name [, ... ] ) ] [ WHERE ( expression ) ] [, ... ] + TABLE [ ONLY ] table_name [ * ] [ [ EXCEPT ] ( column_name [, ... ] ) ] [ WHERE ( expression ) ] [, ... ] TABLES IN SCHEMA { schema_name | CURRENT_SCHEMA } [, ... ] where exception_object is: @@ -260,6 +260,14 @@ ALTER PUBLICATION production_publication ADD ALL TABLES EXCEPT users, department production_publication: ALTER PUBLICATION production_publication ADD TABLE users, departments, TABLES IN SCHEMA production; + + + + Alter publication mypublication to add table + users except column + security_pin: + +ALTER PUBLICATION production_publication ADD TABLE users EXCEPT (security_pin); diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index bddff9ca0cc..0691b102840 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -28,7 +28,7 @@ CREATE PUBLICATION name where publication_object is one of: - TABLE [ ONLY ] table_name [ * ] [ ( column_name [, ... ] ) ] [ WHERE ( expression ) ] [, ... ] + TABLE [ ONLY ] table_name [ * ] [ [ EXCEPT ] ( column_name [, ... ] ) ] [ WHERE ( expression ) ] [, ... ] TABLES IN SCHEMA { schema_name | CURRENT_SCHEMA } [, ... ] where exception_object is: @@ -92,17 +92,24 @@ CREATE PUBLICATION name - When a column list is specified, only the named columns are replicated. - The column list can contain stored generated columns as well. If the - column list is omitted, the publication will replicate all non-generated - columns (including any added in the future) by default. Stored generated - columns can also be replicated if publish_generated_columns - is set to stored. Specifying a column list has no - effect on TRUNCATE commands. See + When a column list without EXCEPT is specified, only + the named columns are replicated. The column list can contain stored + generated columns as well. If the column list is omitted, the publication + will replicate all non-generated columns (including any added in the + future) by default. Stored generated columns can also be replicated if + publish_generated_columns is set to + stored. Specifying a column list has no effect on + TRUNCATE commands. See for details about column lists. + + When a column list is specified with EXCEPT, the named + columns are not replicated. Specifying a column list has no effect on + TRUNCATE commands. + + Only persistent base tables and partitioned tables can be part of a publication. Temporary tables, unlogged tables, foreign tables, @@ -347,10 +354,12 @@ CREATE PUBLICATION name - Any column list must include the REPLICA IDENTITY columns - in order for UPDATE or DELETE - operations to be published. There are no column list restrictions if the - publication publishes only INSERT operations. + In order for UPDATE or DELETE + operations to work, all the REPLICA IDENTITY columns must + be published. So, any column list must name all + REPLICA IDENTITY columns, and any + EXCEPT column list must not name any + REPLICA IDENTITY columns. @@ -373,6 +382,16 @@ CREATE PUBLICATION name to be published. + + The generated columns that are part of the column list specified with the + EXCEPT clause are not published, regardless of the + publish_generated_columns option. However, generated + columns that are not part of the column list specified with the + EXCEPT clause are published according to the value of the + publish_generated_columns option. See + for details. + + The row filter on a table becomes redundant if FOR TABLES IN SCHEMA is specified and the table @@ -494,6 +513,15 @@ CREATE PUBLICATION mypublication FOR ALL TABLES EXCEPT users, departments; CREATE PUBLICATION users_filtered FOR TABLE users (user_id, firstname); + + + Create a publication that publishes all changes for table + users except changes for column + security_pin: + +CREATE PUBLICATION users_safe FOR TABLE users EXCEPT (security_pin); + + diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index af1b8c9ed67..d21b3ff48e1 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -263,14 +263,19 @@ is_schema_publication(Oid pubid) * If a column list is found, the corresponding bitmap is returned through the * cols parameter, if provided. The bitmap is constructed within the given * memory context (mcxt). + * + * If a column list is found specified with EXCEPT clause, except_columns is set + * to true. */ bool check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, - Bitmapset **cols) + Bitmapset **cols, bool *except_columns) { HeapTuple cftuple; bool found = false; + *except_columns = false; + if (pub->alltables) return false; @@ -296,6 +301,16 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, found = true; } + /* Lookup the except attribute */ + cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple, + Anum_pg_publication_rel_prexcept, &isnull); + + if (!isnull) + { + Assert(!pub->alltables); + *except_columns = DatumGetBool(cfdatum); + } + ReleaseSysCache(cftuple); } @@ -657,10 +672,12 @@ pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, MemoryContext mcxt) * Returns a bitmap representing the columns of the specified table. * * Generated columns are included if include_gencols_type is - * PUBLISH_GENCOLS_STORED. + * PUBLISH_GENCOLS_STORED. Columns that are in the except_cols are excluded from + * the column list. */ Bitmapset * -pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type) +pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type, + Bitmapset *except_cols) { Bitmapset *result = NULL; TupleDesc desc = RelationGetDescr(relation); @@ -683,6 +700,9 @@ pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type) continue; } + if (except_cols && bms_is_member(att->attnum, except_cols)) + continue; + result = bms_add_member(result, att->attnum); } @@ -787,8 +807,10 @@ GetRelationPublications(Oid relid, bool except_flag) { HeapTuple tup = &pubrellist->members[i]->tuple; Oid pubid = ((Form_pg_publication_rel) GETSTRUCT(tup))->prpubid; + bool is_except_table = ((Form_pg_publication_rel) GETSTRUCT(tup))->prexcept && + heap_attisnull(tup, Anum_pg_publication_rel_prattrs, NULL); - if (except_flag == ((Form_pg_publication_rel) GETSTRUCT(tup))->prexcept) + if (except_flag == is_except_table) result = lappend_oid(result, pubid); } @@ -828,10 +850,12 @@ GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt, while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_publication_rel pubrel; + bool has_collist = false; pubrel = (Form_pg_publication_rel) GETSTRUCT(tup); + has_collist = !heap_attisnull(tup, Anum_pg_publication_rel_prattrs, NULL); - if (except_flag == pubrel->prexcept) + if (except_flag == (pubrel->prexcept && !has_collist)) result = GetPubPartitionOptionRelations(result, pub_partopt, pubrel->prrelid); @@ -1282,6 +1306,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) Oid schemaid = get_rel_namespace(relid); Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0}; bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0}; + Bitmapset *except_columns = NULL; /* * Form tuple with appropriate data. @@ -1306,11 +1331,29 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) if (HeapTupleIsValid(pubtuple)) { - /* Lookup the column list attribute. */ values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple, Anum_pg_publication_rel_prattrs, &(nulls[2])); + if (!nulls[2]) + { + Datum exceptDatum; + bool isnull; + + /* + * We fetch pubtuple if publication is not FOR ALL TABLES and + * not FOR TABLES IN SCHEMA. So if prexcept is true, it + * indicates that prattrs contains columns to be excluded for + * replication. + */ + exceptDatum = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple, + Anum_pg_publication_rel_prexcept, + &isnull); + + if (!isnull && DatumGetBool(exceptDatum)) + except_columns = pub_collist_to_bitmapset(NULL, values[2], NULL); + } + /* Null indicates no filter. */ values[3] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple, Anum_pg_publication_rel_prqual, @@ -1322,8 +1365,12 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) nulls[3] = true; } - /* Show all columns when the column list is not specified. */ - if (nulls[2]) + /* + * Construct column list to show all columns when no column list is + * specified or to show remaining columns when a column list is + * provided with EXCEPT. + */ + if (except_columns || nulls[2]) { Relation rel = table_open(relid, AccessShareLock); int nattnums = 0; @@ -1354,6 +1401,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) continue; } + /* + * Skip columns that are part of column list specified with + * EXCEPT. + */ + if (except_columns && bms_is_member(att->attnum, except_columns)) + continue; + attnums[nattnums++] = att->attnum; } @@ -1362,6 +1416,11 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) values[2] = PointerGetDatum(buildint2vector(attnums, nattnums)); nulls[2] = false; } + else + { + values[2] = (Datum) 0; + nulls[2] = true; + } table_close(rel, AccessShareLock); } diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index b6d546be291..8813fb28576 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -204,7 +204,6 @@ ObjectsInPublicationToOids(List *pubobjspec_list, ParseState *pstate, switch (pubobj->pubobjtype) { case PUBLICATIONOBJ_TABLE: - pubobj->pubtable->except = false; *rels = lappend(*rels, pubobj->pubtable); break; case PUBLICATIONOBJ_EXCEPT_TABLE: @@ -358,8 +357,8 @@ pub_rf_contains_invalid_column(Oid pubid, Relation relation, List *ancestors, * This function evaluates two conditions: * * 1. Ensures that all columns referenced in the REPLICA IDENTITY are covered - * by the column list. If any column is missing, *invalid_column_list is set - * to true. + * by the column list and are not part of the column list specified with + * EXCEPT. If any column is missing, *invalid_column_list is set to true. * 2. Ensures that all the generated columns referenced in the REPLICA IDENTITY * are published, either by being explicitly named in the column list or, if * no column list is specified, by setting the option @@ -381,6 +380,7 @@ pub_contains_invalid_column(Oid pubid, Relation relation, List *ancestors, TupleDesc desc = RelationGetDescr(relation); Publication *pub; int x; + bool except_columns = false; *invalid_column_list = false; *invalid_gen_col = false; @@ -404,7 +404,8 @@ pub_contains_invalid_column(Oid pubid, Relation relation, List *ancestors, /* Fetch the column list */ pub = GetPublication(pubid); - check_and_fetch_column_list(pub, publish_as_relid, NULL, &columns); + check_and_fetch_column_list(pub, publish_as_relid, NULL, &columns, + &except_columns); if (relation->rd_rel->relreplident == REPLICA_IDENTITY_FULL) { @@ -494,8 +495,14 @@ pub_contains_invalid_column(Oid pubid, Relation relation, List *ancestors, attnum = get_attnum(publish_as_relid, colname); } - /* replica identity column, not covered by the column list */ - *invalid_column_list |= !bms_is_member(attnum, columns); + /* + * Replica identity column, not covered by the column list or is part + * of column list specified with EXCEPT. + */ + if (except_columns) + *invalid_column_list |= bms_is_member(attnum, columns); + else + *invalid_column_list |= !bms_is_member(attnum, columns); if (*invalid_column_list && *invalid_gen_col) break; @@ -1452,6 +1459,7 @@ AlterPublicationTables(AlterPublicationStmt *stmt, HeapTuple tup, HeapTuple rftuple; Node *oldrelwhereclause = NULL; Bitmapset *oldcolumns = NULL; + bool oldexcept = false; /* look up the cache for the old relmap */ rftuple = SearchSysCache2(PUBLICATIONRELMAP, @@ -1465,23 +1473,30 @@ AlterPublicationTables(AlterPublicationStmt *stmt, HeapTuple tup, if (HeapTupleIsValid(rftuple)) { bool isnull = true; - Datum whereClauseDatum; - Datum columnListDatum; + Datum datum; /* Load the WHERE clause for this table. */ - whereClauseDatum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple, - Anum_pg_publication_rel_prqual, - &isnull); + datum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple, + Anum_pg_publication_rel_prqual, + &isnull); if (!isnull) - oldrelwhereclause = stringToNode(TextDatumGetCString(whereClauseDatum)); + oldrelwhereclause = stringToNode(TextDatumGetCString(datum)); /* Transform the int2vector column list to a bitmap. */ - columnListDatum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple, - Anum_pg_publication_rel_prattrs, - &isnull); + datum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple, + Anum_pg_publication_rel_prattrs, + &isnull); + + if (!isnull) + oldcolumns = pub_collist_to_bitmapset(NULL, datum, NULL); + + /* Load the prexcept flag for this table. */ + datum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple, + Anum_pg_publication_rel_prexcept, + &isnull); if (!isnull) - oldcolumns = pub_collist_to_bitmapset(NULL, columnListDatum, NULL); + oldexcept = DatumGetBool(datum); ReleaseSysCache(rftuple); } @@ -1508,13 +1523,14 @@ AlterPublicationTables(AlterPublicationStmt *stmt, HeapTuple tup, * Check if any of the new set of relations matches with the * existing relations in the publication. Additionally, if the * relation has an associated WHERE clause, check the WHERE - * expressions also match. Same for the column list. Drop the - * rest. + * expressions also match. Same for the column list and except + * flag. Drop the rest. */ if (newrelid == oldrelid) { if (equal(oldrelwhereclause, newpubrel->whereClause) && - bms_equal(oldcolumns, newcolumns)) + bms_equal(oldcolumns, newcolumns) && + oldexcept == newpubrel->except) { found = true; break; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 49bac034d17..cebd7c2a3c4 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -527,7 +527,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); OptWhereClause operator_def_arg %type opt_column_and_period_list %type rowsfrom_item rowsfrom_list opt_col_def_list -%type opt_ordinality opt_without_overlaps +%type opt_ordinality opt_without_overlaps opt_except %type ExclusionConstraintList ExclusionConstraintElem %type func_arg_list func_arg_list_opt %type func_arg_expr @@ -4450,6 +4450,11 @@ opt_without_overlaps: | /*EMPTY*/ { $$ = false; } ; +opt_except: + EXCEPT { $$ = true; } + | /*EMPTY*/ { $$ = false; } + ; + opt_column_list: '(' columnList ')' { $$ = $2; } | /*EMPTY*/ { $$ = NIL; } @@ -10759,14 +10764,15 @@ CreatePublicationStmt: * relation_expr here. */ PublicationObjSpec: - TABLE relation_expr opt_column_list OptWhereClause + TABLE relation_expr opt_except opt_column_list OptWhereClause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_TABLE; $$->pubtable = makeNode(PublicationTable); $$->pubtable->relation = $2; - $$->pubtable->columns = $3; - $$->pubtable->whereClause = $4; + $$->pubtable->except = $3; + $$->pubtable->columns = $4; + $$->pubtable->whereClause = $5; $$->location = @1; } | TABLES IN_P SCHEMA ColId @@ -10782,7 +10788,7 @@ PublicationObjSpec: $$->pubobjtype = PUBLICATIONOBJ_TABLES_IN_CUR_SCHEMA; $$->location = @4; } - | ColId opt_column_list OptWhereClause + | ColId opt_except opt_column_list OptWhereClause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_CONTINUATION; @@ -10790,7 +10796,7 @@ PublicationObjSpec: * If either a row filter or column list is specified, create * a PublicationTable object. */ - if ($2 || $3) + if ($2 || $3 || $4) { /* * The OptWhereClause must be stored here but it is @@ -10800,8 +10806,9 @@ PublicationObjSpec: */ $$->pubtable = makeNode(PublicationTable); $$->pubtable->relation = makeRangeVar(NULL, $1, @1); - $$->pubtable->columns = $2; - $$->pubtable->whereClause = $3; + $$->pubtable->except = $2; + $$->pubtable->columns = $3; + $$->pubtable->whereClause = $4; } else { @@ -10809,25 +10816,27 @@ PublicationObjSpec: } $$->location = @1; } - | ColId indirection opt_column_list OptWhereClause + | ColId indirection opt_except opt_column_list OptWhereClause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_CONTINUATION; $$->pubtable = makeNode(PublicationTable); $$->pubtable->relation = makeRangeVarFromQualifiedName($1, $2, @1, yyscanner); - $$->pubtable->columns = $3; - $$->pubtable->whereClause = $4; + $$->pubtable->except = $3; + $$->pubtable->columns = $4; + $$->pubtable->whereClause = $5; $$->location = @1; } /* grammar like tablename * , ONLY tablename, ONLY ( tablename ) */ - | extended_relation_expr opt_column_list OptWhereClause + | extended_relation_expr opt_except opt_column_list OptWhereClause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_CONTINUATION; $$->pubtable = makeNode(PublicationTable); $$->pubtable->relation = $1; - $$->pubtable->columns = $2; - $$->pubtable->whereClause = $3; + $$->pubtable->except = $2; + $$->pubtable->columns = $3; + $$->pubtable->whereClause = $4; } | CURRENT_SCHEMA { @@ -19695,6 +19704,13 @@ preprocess_pubobj_list(List *pubobjspec_list, core_yyscan_t yyscanner) errmsg("invalid table name"), parser_errposition(pubobj->location)); + if (pubobj->pubtable && pubobj->pubtable->except && + pubobj->pubtable->columns == NULL) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("table without column list cannot use EXCEPT clause"), + parser_errposition(pubobj->location)); + if (pubobj->name) { /* convert it to PublicationTable */ diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index d3356bc84ee..68ff559e80c 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -821,10 +821,18 @@ copy_read_data(void *outbuf, int minread, int maxread) * This function also returns (a) the relation qualifications to be used in * the COPY command, and (b) whether the remote relation has published any * generated column. + * + * With the introduction of the EXCEPT qualifier in column lists, it is now + * possible to define a publication that excludes all columns of a table. When + * the column list is fetched from the remote server and is NULL, it normally + * indicates that all columns are included. To distinguish this from the case + * where all columns are explicitly excluded, the 'all_cols_excluded' flag has + * been introduced. */ static void fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, - List **qual, bool *gencol_published) + List **qual, bool *gencol_published, + bool *all_cols_excluded) { WalRcvExecResult *res; StringInfoData cmd; @@ -838,6 +846,9 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, Bitmapset *included_cols = NULL; int server_version = walrcv_server_version(LogRepWorkerWalRcvConn); + Assert(*gencol_published == false); + Assert(*all_cols_excluded == false); + lrel->nspname = nspname; lrel->relname = relname; @@ -888,7 +899,7 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, { WalRcvExecResult *pubres; TupleTableSlot *tslot; - Oid attrsRow[] = {INT2VECTOROID}; + Oid attrsRow[] = {INT2VECTOROID, BOOLOID}; /* Build the pub_names comma-separated string. */ pub_names = makeStringInfo(); @@ -902,7 +913,17 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, appendStringInfo(&cmd, "SELECT DISTINCT" " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)" - " THEN NULL ELSE gpt.attrs END)" + " THEN NULL ELSE gpt.attrs END)"); + + /* + * When publication is created with EXCEPT (column-list) and all + * columns are specified, gpt.attrs will be NULL and no columns are + * published in this case. + */ + if (server_version >= 190000) + appendStringInfo(&cmd, ", gpt.attrs IS NULL AND c.relnatts > 0"); + + appendStringInfo(&cmd, " FROM pg_publication p," " LATERAL pg_get_publication_tables(p.pubname) gpt," " pg_class c" @@ -912,7 +933,7 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, pub_names->data); pubres = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, - lengthof(attrsRow), attrsRow); + server_version >= 190000 ? 2 : 1, attrsRow); if (pubres->status != WALRCV_OK_TUPLES) ereport(ERROR, @@ -959,6 +980,9 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, included_cols = bms_add_member(included_cols, elems[natt]); } + if (server_version >= 190000) + *all_cols_excluded = DatumGetBool(slot_getattr(tslot, 2, &isnull)); + ExecClearTuple(tslot); } ExecDropSingleTupleTableSlot(tslot); @@ -1021,7 +1045,8 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, Assert(!isnull); /* If the column is not in the column list, skip it. */ - if (included_cols != NULL && !bms_is_member(attnum, included_cols)) + if (*all_cols_excluded || + (included_cols != NULL && !bms_is_member(attnum, included_cols))) { ExecClearTuple(slot); continue; @@ -1153,11 +1178,15 @@ copy_table(Relation rel) ParseState *pstate; List *options = NIL; bool gencol_published = false; + bool all_cols_excluded = false; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), RelationGetRelationName(rel), &lrel, &qual, - &gencol_published); + &gencol_published, &all_cols_excluded); + + if (all_cols_excluded) + return; /* Put the relation into relmap. */ logicalrep_relmap_update(&lrel); diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index b9d676d1f18..1cf90f1875d 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -185,6 +185,16 @@ typedef struct RelationSyncEntry * row filter expressions, column list, etc. */ MemoryContext entry_cxt; + + /* + * Indicates whether no columns are published for a given relation. With + * the introduction of the EXCEPT qualifier in column lists, it is now + * possible to define a publication that excludes all columns of a table. + * However, the 'columns' attribute cannot represent this case, since a + * NULL value implies that all columns are published. To distinguish this + * scenario, the 'all_cols_excluded' flag is introduced. + */ + bool all_cols_excluded; } RelationSyncEntry; /* @@ -1068,12 +1078,21 @@ check_and_init_gencol(PGOutputData *data, List *publications, */ foreach_ptr(Publication, pub, publications) { + bool has_column_list = false; + bool except_columns = false; + + has_column_list = check_and_fetch_column_list(pub, + entry->publish_as_relid, + NULL, NULL, + &except_columns); + /* * The column list takes precedence over the * 'publish_generated_columns' parameter. Those will be checked later, - * see pgoutput_column_list_init. + * see pgoutput_column_list_init. But when a column list is specified + * with EXCEPT, it should be checked. */ - if (check_and_fetch_column_list(pub, entry->publish_as_relid, NULL, NULL)) + if (has_column_list && !except_columns) continue; if (first) @@ -1122,19 +1141,41 @@ pgoutput_column_list_init(PGOutputData *data, List *publications, { Publication *pub = lfirst(lc); Bitmapset *cols = NULL; + bool except_columns = false; + bool all_cols_excluded = false; /* Retrieve the bitmap of columns for a column list publication. */ found_pub_collist |= check_and_fetch_column_list(pub, entry->publish_as_relid, - entry->entry_cxt, &cols); + entry->entry_cxt, &cols, + &except_columns); + + /* + * If column list is specified with EXCEPT retrieve bitmap of columns + * which are not part of this column list. + */ + if (except_columns) + { + MemoryContext oldcxt = MemoryContextSwitchTo(entry->entry_cxt); + + cols = pub_form_cols_map(relation, + entry->include_gencols_type, cols); + MemoryContextSwitchTo(oldcxt); + + if (!cols) + all_cols_excluded = true; + } /* - * For non-column list publications — e.g. TABLE (without a column - * list), ALL TABLES, or ALL TABLES IN SCHEMA, we consider all columns - * of the table (including generated columns when + * If 'cols' is null, it indicates that the publication is either a + * non-column list publication or one where all columns are excluded. + * When 'all_cols_excluded' is true, it explicitly means all columns + * have been excluded. For non-column list publications — e.g. TABLE + * (without a column list), ALL TABLES, or ALL TABLES IN SCHEMA, we + * consider all columns of the table (including generated columns when * 'publish_generated_columns' parameter is true). */ - if (!cols) + if (!all_cols_excluded && !cols) { /* * Cache the table columns for the first publication with no @@ -1146,7 +1187,7 @@ pgoutput_column_list_init(PGOutputData *data, List *publications, MemoryContext oldcxt = MemoryContextSwitchTo(entry->entry_cxt); relcols = pub_form_cols_map(relation, - entry->include_gencols_type); + entry->include_gencols_type, NULL); MemoryContextSwitchTo(oldcxt); } @@ -1156,9 +1197,11 @@ pgoutput_column_list_init(PGOutputData *data, List *publications, if (first) { entry->columns = cols; + entry->all_cols_excluded = all_cols_excluded; first = false; } - else if (!bms_equal(entry->columns, cols)) + else if ((entry->all_cols_excluded != all_cols_excluded) || + !bms_equal(entry->columns, cols)) ereport(ERROR, errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot use different column lists for table \"%s.%s\" in different publications", @@ -1482,6 +1525,13 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, relentry = get_rel_sync_entry(data, relation); + /* + * If all columns of a table are present in column list specified with + * EXCEPT, skip publishing the changes. + */ + if (relentry->all_cols_excluded) + return; + /* First check the table filter */ switch (action) { @@ -2059,6 +2109,7 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) entry->publish_as_relid = InvalidOid; entry->columns = NULL; entry->attrmap = NULL; + entry->all_cols_excluded = false; } /* Validate the entry */ @@ -2108,6 +2159,7 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) entry->pubactions.pubupdate = false; entry->pubactions.pubdelete = false; entry->pubactions.pubtruncate = false; + entry->all_cols_excluded = false; /* * Tuple slots cleanups. (Will be rebuilt later if needed). diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 105031a5cbc..a57ba69f748 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4865,24 +4865,7 @@ getPublicationTables(Archive *fout, TableInfo tblinfo[], int numTables) if (tbinfo == NULL) continue; - /* OK, make a DumpableObject for this relationship */ - if (strcmp(prexcept, "f") == 0) - pubrinfo[j].dobj.objType = DO_PUBLICATION_REL; - else - pubrinfo[j].dobj.objType = DO_PUBLICATION_EXCEPT_REL; - - pubrinfo[j].dobj.catId.tableoid = - atooid(PQgetvalue(res, i, i_tableoid)); - pubrinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid)); - AssignDumpId(&pubrinfo[j].dobj); - pubrinfo[j].dobj.namespace = tbinfo->dobj.namespace; - pubrinfo[j].dobj.name = tbinfo->dobj.name; - pubrinfo[j].publication = pubinfo; - pubrinfo[j].pubtable = tbinfo; - if (PQgetisnull(res, i, i_prrelqual)) - pubrinfo[j].pubrelqual = NULL; - else - pubrinfo[j].pubrelqual = pg_strdup(PQgetvalue(res, i, i_prrelqual)); + pubrinfo[j].pubexcept = (strcmp(prexcept, "t") == 0); if (!PQgetisnull(res, i, i_prattrs)) { @@ -4908,10 +4891,29 @@ getPublicationTables(Archive *fout, TableInfo tblinfo[], int numTables) else pubrinfo[j].pubrattrs = NULL; + /* OK, make a DumpableObject for this relationship */ + if (pubrinfo[j].pubexcept && !pubrinfo[j].pubrattrs) + pubrinfo[j].dobj.objType = DO_PUBLICATION_EXCEPT_REL; + else + pubrinfo[j].dobj.objType = DO_PUBLICATION_REL; + + pubrinfo[j].dobj.catId.tableoid = + atooid(PQgetvalue(res, i, i_tableoid)); + pubrinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid)); + AssignDumpId(&pubrinfo[j].dobj); + pubrinfo[j].dobj.namespace = tbinfo->dobj.namespace; + pubrinfo[j].dobj.name = tbinfo->dobj.name; + pubrinfo[j].publication = pubinfo; + pubrinfo[j].pubtable = tbinfo; + if (PQgetisnull(res, i, i_prrelqual)) + pubrinfo[j].pubrelqual = NULL; + else + pubrinfo[j].pubrelqual = pg_strdup(PQgetvalue(res, i, i_prrelqual)); + /* Decide whether we want to dump it */ selectDumpablePublicationObject(&(pubrinfo[j].dobj), fout); - if (strcmp(prexcept, "t") == 0) + if (pubrinfo[j].pubexcept && !pubrinfo[j].pubrattrs) simple_ptr_list_append(&exceptinfo, &pubrinfo[j]); j++; @@ -4991,7 +4993,12 @@ dumpPublicationTable(Archive *fout, const PublicationRelInfo *pubrinfo) fmtQualifiedDumpable(tbinfo)); if (pubrinfo->pubrattrs) + { + if (pubrinfo->pubexcept) + appendPQExpBufferStr(query, " EXCEPT"); + appendPQExpBuffer(query, " (%s)", pubrinfo->pubrattrs); + } if (pubrinfo->pubrelqual) { diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index c178edb4e05..5e2aa1b0cf0 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -689,6 +689,7 @@ typedef struct _PublicationRelInfo TableInfo *pubtable; char *pubrelqual; char *pubrattrs; + bool pubexcept; } PublicationRelInfo; /* diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index f3fe4ab30f8..6b727458076 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1560,6 +1560,68 @@ describeTableDetails(const char *pattern, bool verbose, bool showSystem) return true; } +/* + * If is_tbl_desc is true add footer to table description else add footer to + * publication description. + */ +static bool +addFooterToPublicationOrTableDesc(PQExpBuffer buf, const char *footermsg, + bool as_schema, printTableContent *const cont, + bool is_tbl_desc) +{ + PGresult *res; + int count = 0; + int i = 0; + int col = is_tbl_desc ? 0 : 1; + + res = PSQLexec(buf->data); + if (!res) + return false; + else + count = PQntuples(res); + + if (count > 0) + printTableAddFooter(cont, footermsg); + + /*--------------------------------------------------- + * Publication/ table description columns: + * [0]: schema name (nspname) + * [col]: table name (relname) / publication name (pubname) + * [col + 1]: row filter expression (prqual), may be NULL + * [col + 2]: column list (comma-separated), may be NULL + * [col + 3]: except flag ("t" if EXCEPT, else "f") + *--------------------------------------------------- + */ + for (i = 0; i < count; i++) + { + if (as_schema) + printfPQExpBuffer(buf, " \"%s\"", PQgetvalue(res, i, 0)); + else + { + if (is_tbl_desc) + printfPQExpBuffer(buf, " \"%s\"", PQgetvalue(res, i, col)); + else + printfPQExpBuffer(buf, " \"%s.%s\"", PQgetvalue(res, i, 0), + PQgetvalue(res, i, col)); + + if (!PQgetisnull(res, i, col + 2)) + { + if (strcmp(PQgetvalue(res, i, col + 3), "t") == 0) + appendPQExpBuffer(buf, " EXCEPT"); + appendPQExpBuffer(buf, " (%s)", PQgetvalue(res, i, col + 2)); + } + + if (!PQgetisnull(res, i, col + 1)) + appendPQExpBuffer(buf, " WHERE %s", PQgetvalue(res, i, col + 1)); + } + + printTableAddFooter(cont, buf->data); + } + + PQclear(res); + return true; +} + /* * describeOneTableDetails (for \d) * @@ -3019,16 +3081,27 @@ describeOneTableDetails(const char *schemaname, /* print any publications */ if (pset.sversion >= 100000) { - if (pset.sversion >= 150000) + if (pset.sversion >= 190000) { printfPQExpBuffer(&buf, + + /* + * Get all publications for the schema that this relation is + * part of + */ "SELECT pubname\n" " , NULL\n" " , NULL\n" + " , NULL\n" "FROM pg_catalog.pg_publication p\n" " JOIN pg_catalog.pg_publication_namespace pn ON p.oid = pn.pnpubid\n" " JOIN pg_catalog.pg_class pc ON pc.relnamespace = pn.pnnspid\n" "WHERE pc.oid ='%s' and pg_catalog.pg_relation_is_publishable('%s')\n" + + /* + * Get all publications for this relation created using FOR + * TABLE + */ "UNION\n" "SELECT pubname\n" " , pg_get_expr(pr.prqual, c.oid)\n" @@ -3038,35 +3111,67 @@ describeOneTableDetails(const char *schemaname, " pg_catalog.pg_attribute\n" " WHERE attrelid = pr.prrelid AND attnum = prattrs[s])\n" " ELSE NULL END) " + " , prexcept " "FROM pg_catalog.pg_publication p\n" " JOIN pg_catalog.pg_publication_rel pr ON p.oid = pr.prpubid\n" " JOIN pg_catalog.pg_class c ON c.oid = pr.prrelid\n" - "WHERE pr.prrelid = '%s'\n", - oid, oid, oid); - - if (pset.sversion >= 190000) - appendPQExpBufferStr(&buf, " AND NOT pr.prexcept\n"); + "WHERE pr.prrelid = '%s' " + "AND p.puballtables = false\n" + "AND c.relnamespace NOT IN (\n " + " SELECT pnnspid FROM\n" + " pg_catalog.pg_publication_namespace)\n" - appendPQExpBuffer(&buf, + /* + * Get all FOR ALL TABLES publications that include this + * relation + */ "UNION\n" "SELECT pubname\n" " , NULL\n" " , NULL\n" + " , NULL\n" "FROM pg_catalog.pg_publication p\n" - "WHERE p.puballtables AND pg_catalog.pg_relation_is_publishable('%s')\n", - oid); - - if (pset.sversion >= 190000) - appendPQExpBuffer(&buf, - " AND NOT EXISTS (\n" - " SELECT 1\n" - " FROM pg_catalog.pg_publication_rel pr\n" - " JOIN pg_catalog.pg_class pc\n" - " ON pr.prrelid = pc.oid\n" - " WHERE pr.prrelid = '%s' AND pr.prpubid = p.oid)\n", - oid); - - appendPQExpBufferStr(&buf, "ORDER BY 1;"); + "WHERE p.puballtables AND pg_catalog.pg_relation_is_publishable('%s')\n" + " AND NOT EXISTS (\n" + " SELECT 1\n" + " FROM pg_catalog.pg_publication_rel pr\n" + " JOIN pg_catalog.pg_class pc\n" + " ON pr.prrelid = pc.oid\n" + " WHERE pr.prrelid = '%s' AND pr.prpubid = p.oid)\n" + "ORDER BY 1;", + oid, oid, oid, oid, oid); + } + else if (pset.sversion >= 150000) + { + printfPQExpBuffer(&buf, + "SELECT pubname\n" + " , NULL\n" + " , NULL\n" + "FROM pg_catalog.pg_publication p\n" + " JOIN pg_catalog.pg_publication_namespace pn ON p.oid = pn.pnpubid\n" + " JOIN pg_catalog.pg_class pc ON pc.relnamespace = pn.pnnspid\n" + "WHERE pc.oid ='%s' and pg_catalog.pg_relation_is_publishable('%s')\n" + "UNION\n" + "SELECT pubname\n" + " , pg_get_expr(pr.prqual, c.oid)\n" + " , (CASE WHEN pr.prattrs IS NOT NULL THEN\n" + " (SELECT string_agg(attname, ', ')\n" + " FROM pg_catalog.generate_series(0, pg_catalog.array_upper(pr.prattrs::pg_catalog.int2[], 1)) s,\n" + " pg_catalog.pg_attribute\n" + " WHERE attrelid = pr.prrelid AND attnum = prattrs[s])\n" + " ELSE NULL END) " + "FROM pg_catalog.pg_publication p\n" + " JOIN pg_catalog.pg_publication_rel pr ON p.oid = pr.prpubid\n" + " JOIN pg_catalog.pg_class c ON c.oid = pr.prrelid\n" + "WHERE pr.prrelid = '%s'\n" + "UNION\n" + "SELECT pubname\n" + " , NULL\n" + " , NULL\n" + "FROM pg_catalog.pg_publication p\n" + "WHERE p.puballtables AND pg_catalog.pg_relation_is_publishable('%s')\n" + "ORDER BY 1;", + oid, oid, oid, oid); } else { @@ -3087,34 +3192,8 @@ describeOneTableDetails(const char *schemaname, oid, oid); } - result = PSQLexec(buf.data); - if (!result) + if (!addFooterToPublicationOrTableDesc(&buf, _("Publications:"), false, &cont, true)) goto error_return; - else - tuples = PQntuples(result); - - if (tuples > 0) - printTableAddFooter(&cont, _("Publications:")); - - /* Might be an empty set - that's ok */ - for (i = 0; i < tuples; i++) - { - printfPQExpBuffer(&buf, " \"%s\"", - PQgetvalue(result, i, 0)); - - /* column list (if any) */ - if (!PQgetisnull(result, i, 2)) - appendPQExpBuffer(&buf, " (%s)", - PQgetvalue(result, i, 2)); - - /* row filter (if any) */ - if (!PQgetisnull(result, i, 1)) - appendPQExpBuffer(&buf, " WHERE %s", - PQgetvalue(result, i, 1)); - - printTableAddFooter(&cont, buf.data); - } - PQclear(result); } /* @@ -6491,49 +6570,6 @@ listPublications(const char *pattern) return true; } -/* - * Add footer to publication description. - */ -static bool -addFooterToPublicationDesc(PQExpBuffer buf, const char *footermsg, - bool as_schema, printTableContent *const cont) -{ - PGresult *res; - int count = 0; - int i = 0; - - res = PSQLexec(buf->data); - if (!res) - return false; - else - count = PQntuples(res); - - if (count > 0) - printTableAddFooter(cont, footermsg); - - for (i = 0; i < count; i++) - { - if (as_schema) - printfPQExpBuffer(buf, " \"%s\"", PQgetvalue(res, i, 0)); - else - { - printfPQExpBuffer(buf, " \"%s.%s\"", PQgetvalue(res, i, 0), - PQgetvalue(res, i, 1)); - - if (!PQgetisnull(res, i, 3)) - appendPQExpBuffer(buf, " (%s)", PQgetvalue(res, i, 3)); - - if (!PQgetisnull(res, i, 2)) - appendPQExpBuffer(buf, " WHERE %s", PQgetvalue(res, i, 2)); - } - - printTableAddFooter(cont, buf->data); - } - - PQclear(res); - return true; -} - /* * \dRp+ * Describes publications including the contents. @@ -6704,6 +6740,12 @@ describePublications(const char *pattern) else appendPQExpBufferStr(&buf, ", NULL, NULL"); + + if (pset.sversion >= 190000) + appendPQExpBufferStr(&buf, ", prexcept"); + else + appendPQExpBufferStr(&buf, ", NULL"); + appendPQExpBuffer(&buf, "\nFROM pg_catalog.pg_class c,\n" " pg_catalog.pg_namespace n,\n" @@ -6712,11 +6754,8 @@ describePublications(const char *pattern) " AND c.oid = pr.prrelid\n" " AND pr.prpubid = '%s'\n", pubid); - if (pset.sversion >= 190000) - appendPQExpBuffer(&buf, " AND NOT pr.prexcept\n"); - appendPQExpBuffer(&buf, "ORDER BY 1,2"); - if (!addFooterToPublicationDesc(&buf, _("Tables:"), false, &cont)) + if (!addFooterToPublicationOrTableDesc(&buf, _("Tables:"), false, &cont, false)) goto error_return; if (pset.sversion >= 150000) @@ -6728,8 +6767,8 @@ describePublications(const char *pattern) " JOIN pg_catalog.pg_publication_namespace pn ON n.oid = pn.pnnspid\n" "WHERE pn.pnpubid = '%s'\n" "ORDER BY 1", pubid); - if (!addFooterToPublicationDesc(&buf, _("Tables from schemas:"), - true, &cont)) + if (!addFooterToPublicationOrTableDesc(&buf, _("Tables from schemas:"), + true, &cont, false)) goto error_return; } } @@ -6745,8 +6784,8 @@ describePublications(const char *pattern) "WHERE pr.prpubid = '%s'\n" " AND pr.prexcept\n" "ORDER BY 1", pubid); - if (!addFooterToPublicationDesc(&buf, _("Except tables:"), - true, &cont)) + if (!addFooterToPublicationOrTableDesc(&buf, _("Except tables:"), + true, &cont, false)) goto error_return; } } diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 5efdcf56347..dd560c9ba8c 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -2272,6 +2272,8 @@ match_previous_words(int pattern_id, COMPLETE_WITH("ALL TABLES", "TABLES IN SCHEMA", "TABLE"); else if (Matches("ALTER", "PUBLICATION", MatchAny, "ADD", "ALL", "TABLES")) COMPLETE_WITH("EXCEPT TABLE"); + else if (Matches("ALTER", "PUBLICATION", MatchAny, "ADD|SET", "TABLE", MatchAny)) + COMPLETE_WITH("EXCEPT ("); else if (Matches("ALTER", "PUBLICATION", MatchAny, "ADD|SET", "TABLE")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables); else if (Matches("ALTER", "PUBLICATION", MatchAny, "ADD", "ALL", "TABLES", "EXCEPT", "TABLE")) @@ -3601,7 +3603,7 @@ match_previous_words(int pattern_id, else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES")) COMPLETE_WITH("IN SCHEMA"); else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLE", MatchAny) && !ends_with(prev_wd, ',')) - COMPLETE_WITH("WHERE (", "WITH ("); + COMPLETE_WITH("EXCEPT (", "WHERE (", "WITH ("); /* Complete "CREATE PUBLICATION FOR TABLE" with ", ..." */ else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLE")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables); diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 022467fb45c..2a1dc48ccb8 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -181,7 +181,8 @@ extern Oid GetTopMostAncestorInPublication(Oid puboid, List *ancestors, extern bool is_publishable_relation(Relation rel); extern bool is_schema_publication(Oid pubid); extern bool check_and_fetch_column_list(Publication *pub, Oid relid, - MemoryContext mcxt, Bitmapset **cols); + MemoryContext mcxt, Bitmapset **cols, + bool *except_columns); extern ObjectAddress publication_add_relation(Oid pubid, PublicationRelInfo *pri, bool if_not_exists); extern Bitmapset *pub_collist_validate(Relation targetrel, List *columns); @@ -191,6 +192,7 @@ extern ObjectAddress publication_add_schema(Oid pubid, Oid schemaid, extern Bitmapset *pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, MemoryContext mcxt); extern Bitmapset *pub_form_cols_map(Relation relation, - PublishGencolsType include_gencols_type); + PublishGencolsType include_gencols_type, + Bitmapset *except_cols); #endif /* PG_PUBLICATION_H */ diff --git a/src/include/catalog/pg_publication_rel.h b/src/include/catalog/pg_publication_rel.h index e7d7f3ba85c..6a2168fc32c 100644 --- a/src/include/catalog/pg_publication_rel.h +++ b/src/include/catalog/pg_publication_rel.h @@ -31,11 +31,12 @@ CATALOG(pg_publication_rel,6106,PublicationRelRelationId) Oid oid; /* oid */ Oid prpubid BKI_LOOKUP(pg_publication); /* Oid of the publication */ Oid prrelid BKI_LOOKUP(pg_class); /* Oid of the relation */ - bool prexcept BKI_DEFAULT(f); /* exclude the relation */ + bool prexcept BKI_DEFAULT(f); /* exclude the relation or columns */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ pg_node_tree prqual; /* qualifications */ - int2vector prattrs; /* columns to replicate */ + int2vector prattrs; /* columns to replicate or exclude to + * replicate */ #endif } FormData_pg_publication_rel; diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 49afeb77622..69404c6aa1f 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -2125,6 +2125,94 @@ SET ROLE regress_publication_user; DROP PUBLICATION testpub_reset; DROP TABLE pub_sch1.tbl1; DROP TABLE pub_sch1.tbl2; +-- ====================================================== +-- Test EXCEPT columns for CREATE PUBLICATION +SET client_min_messages = 'ERROR'; +CREATE TABLE pub_test_except1 (a int NOT NULL, b int, c int NOT NULL, d int); +CREATE TABLE pub_sch1.pub_test_except2 (a int, b int, c int, d int); +-- Verify that publication is created with EXCEPT +CREATE PUBLICATION testpub_except FOR TABLE pub_test_except1, pub_sch1.pub_test_except2 EXCEPT (b, c); +SELECT * FROM pg_publication_tables WHERE pubname = 'testpub_except'; + pubname | schemaname | tablename | attnames | rowfilter +----------------+------------+------------------+-----------+----------- + testpub_except | public | pub_test_except1 | {a,b,c,d} | + testpub_except | pub_sch1 | pub_test_except2 | {a,d} | +(2 rows) + +-- Cannot use EXCEPT col-lists combined with TABLES IN SCHEMA +CREATE PUBLICATION testpub_except2 FOR TABLES IN SCHEMA pub_sch1, TABLE pub_test_except1 EXCEPT (b, c); +ERROR: cannot use column list for relation "public.pub_test_except1" in publication "testpub_except2" +DETAIL: Column lists cannot be specified in publications containing FOR TABLES IN SCHEMA elements. +-- Syntax error EXCEPT without a col-list +CREATE PUBLICATION testpub_except2 FOR TABLE pub_test_except1 EXCEPT; +ERROR: table without column list cannot use EXCEPT clause +LINE 1: CREATE PUBLICATION testpub_except2 FOR TABLE pub_test_except... + ^ +-- Verify ok - ALTER PUBLICATION ... SET ... EXCEPT (col-list) +ALTER PUBLICATION testpub_except SET TABLE pub_test_except1 EXCEPT (a, b), pub_sch1.pub_test_except2; +\dRp+ testpub_except + Publication testpub_except + Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | t | t | t | t | none | f +Tables: + "pub_sch1.pub_test_except2" + "public.pub_test_except1" EXCEPT (a, b) + +-- Verify fails - ALTER PUBLICATION ... DROP ... EXCEPT (col-list) +ALTER PUBLICATION testpub_except DROP TABLE pub_test_except1 EXCEPT (a, b); +ERROR: column list must not be specified in ALTER PUBLICATION ... DROP +-- Verify ok - ALTER PUBLICATION ... DROP +ALTER PUBLICATION testpub_except DROP TABLE pub_test_except1; +-- Verify ok - ALTER PUBLICATION ... ADD ... EXCEPT (col-list) +ALTER PUBLICATION testpub_except ADD TABLE pub_test_except1 EXCEPT (c, d); +\dRp+ testpub_except + Publication testpub_except + Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | t | t | t | t | none | f +Tables: + "pub_sch1.pub_test_except2" + "public.pub_test_except1" EXCEPT (c, d) + +-- Verify fails - EXCEPT col-list cannot contain RI cols (when using RI FULL) +ALTER TABLE pub_test_except1 REPLICA IDENTITY FULL; +UPDATE pub_test_except1 SET a = 3 WHERE a = 1; +ERROR: cannot update table "pub_test_except1" +DETAIL: Column list used by the publication does not cover the replica identity. +-- Verify fails - EXCEPT col-list cannot contain RI cols (when using INDEX) +CREATE UNIQUE INDEX pub_test_except1_ac_idx ON pub_test_except1 (a, c); +ALTER TABLE pub_test_except1 REPLICA IDENTITY USING INDEX pub_test_except1_ac_idx; +UPDATE pub_test_except1 SET a = 3 WHERE a = 1; +ERROR: cannot update table "pub_test_except1" +DETAIL: Column list used by the publication does not cover the replica identity. +DROP INDEX pub_test_except1_ac_idx; +-- Verify ok - no clash between RI cols and the EXCEPT col-list +CREATE UNIQUE INDEX pub_test_except1_a_idx ON pub_test_except1 (a); +ALTER TABLE pub_test_except1 REPLICA IDENTITY USING INDEX pub_test_except1_a_idx; +UPDATE pub_test_except1 SET a = 3 WHERE a = 1; +-- Verify description of a table with publication with EXCEPT col-list +\d+ pub_test_except1 + Table "public.pub_test_except1" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | not null | | plain | | + b | integer | | | | plain | | + c | integer | | not null | | plain | | + d | integer | | | | plain | | +Indexes: + "pub_test_except1_a_idx" UNIQUE, btree (a) REPLICA IDENTITY +Publications: + "testpub_except" EXCEPT (c, d) +Not-null constraints: + "pub_test_except1_a_not_null" NOT NULL "a" + "pub_test_except1_c_not_null" NOT NULL "c" + +-- cleanup +DROP INDEX pub_test_except1_a_idx; +DROP PUBLICATION testpub_except; +DROP TABLE pub_test_except1; +DROP TABLE pub_sch1.pub_test_except2; DROP SCHEMA pub_sch1; RESET client_min_messages; RESET SESSION AUTHORIZATION; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 6240cd97ce3..bf64e8a3ce1 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1318,6 +1318,61 @@ SET ROLE regress_publication_user; DROP PUBLICATION testpub_reset; DROP TABLE pub_sch1.tbl1; DROP TABLE pub_sch1.tbl2; + +-- ====================================================== +-- Test EXCEPT columns for CREATE PUBLICATION + +SET client_min_messages = 'ERROR'; +CREATE TABLE pub_test_except1 (a int NOT NULL, b int, c int NOT NULL, d int); +CREATE TABLE pub_sch1.pub_test_except2 (a int, b int, c int, d int); + +-- Verify that publication is created with EXCEPT +CREATE PUBLICATION testpub_except FOR TABLE pub_test_except1, pub_sch1.pub_test_except2 EXCEPT (b, c); +SELECT * FROM pg_publication_tables WHERE pubname = 'testpub_except'; + +-- Cannot use EXCEPT col-lists combined with TABLES IN SCHEMA +CREATE PUBLICATION testpub_except2 FOR TABLES IN SCHEMA pub_sch1, TABLE pub_test_except1 EXCEPT (b, c); + +-- Syntax error EXCEPT without a col-list +CREATE PUBLICATION testpub_except2 FOR TABLE pub_test_except1 EXCEPT; + +-- Verify ok - ALTER PUBLICATION ... SET ... EXCEPT (col-list) +ALTER PUBLICATION testpub_except SET TABLE pub_test_except1 EXCEPT (a, b), pub_sch1.pub_test_except2; +\dRp+ testpub_except + +-- Verify fails - ALTER PUBLICATION ... DROP ... EXCEPT (col-list) +ALTER PUBLICATION testpub_except DROP TABLE pub_test_except1 EXCEPT (a, b); + +-- Verify ok - ALTER PUBLICATION ... DROP +ALTER PUBLICATION testpub_except DROP TABLE pub_test_except1; + +-- Verify ok - ALTER PUBLICATION ... ADD ... EXCEPT (col-list) +ALTER PUBLICATION testpub_except ADD TABLE pub_test_except1 EXCEPT (c, d); +\dRp+ testpub_except + +-- Verify fails - EXCEPT col-list cannot contain RI cols (when using RI FULL) +ALTER TABLE pub_test_except1 REPLICA IDENTITY FULL; +UPDATE pub_test_except1 SET a = 3 WHERE a = 1; + +-- Verify fails - EXCEPT col-list cannot contain RI cols (when using INDEX) +CREATE UNIQUE INDEX pub_test_except1_ac_idx ON pub_test_except1 (a, c); +ALTER TABLE pub_test_except1 REPLICA IDENTITY USING INDEX pub_test_except1_ac_idx; +UPDATE pub_test_except1 SET a = 3 WHERE a = 1; +DROP INDEX pub_test_except1_ac_idx; + +-- Verify ok - no clash between RI cols and the EXCEPT col-list +CREATE UNIQUE INDEX pub_test_except1_a_idx ON pub_test_except1 (a); +ALTER TABLE pub_test_except1 REPLICA IDENTITY USING INDEX pub_test_except1_a_idx; +UPDATE pub_test_except1 SET a = 3 WHERE a = 1; + +-- Verify description of a table with publication with EXCEPT col-list +\d+ pub_test_except1 + +-- cleanup +DROP INDEX pub_test_except1_a_idx; +DROP PUBLICATION testpub_except; +DROP TABLE pub_test_except1; +DROP TABLE pub_sch1.pub_test_except2; DROP SCHEMA pub_sch1; RESET client_min_messages; diff --git a/src/test/subscription/meson.build b/src/test/subscription/meson.build index e8a117f3421..e010de1e1d0 100644 --- a/src/test/subscription/meson.build +++ b/src/test/subscription/meson.build @@ -43,6 +43,7 @@ tests += { 't/034_temporal.pl', 't/035_conflicts.pl', 't/036_rep_changes_except_table.pl', + 't/037_rep_changes_except_collist.pl', 't/100_bugs.pl', ], }, diff --git a/src/test/subscription/t/037_rep_changes_except_collist.pl b/src/test/subscription/t/037_rep_changes_except_collist.pl new file mode 100644 index 00000000000..79e63c0f449 --- /dev/null +++ b/src/test/subscription/t/037_rep_changes_except_collist.pl @@ -0,0 +1,190 @@ +# Copyright (c) 2021-2025, PostgreSQL Global Development Group + +# Logical replication tests for EXCEPT (column-list) publications +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Initial setup +$node_publisher->safe_psql( + 'postgres', qq ( + CREATE SCHEMA sch1; + CREATE TABLE tab1 (a int, b int NOT NULL, c int); + CREATE TABLE sch1.tab1 (a int, b int, c int); + CREATE TABLE tab2 (a int, b int, c int); + CREATE TABLE tab3 (a int, bgen int GENERATED ALWAYS AS (a * 2) STORED, cgen int GENERATED ALWAYS AS (a * 3) STORED); + CREATE TABLE tab4 (a int, bgen int GENERATED ALWAYS AS (a * 2) STORED, cgen int GENERATED ALWAYS AS (a * 3) STORED); + CREATE TABLE tab5 (a int, b int, c int); + CREATE TABLE tab6 (agen int GENERATED ALWAYS AS (1) STORED, bgen int GENERATED ALWAYS AS (2) STORED); + INSERT INTO tab1 VALUES (1, 2, 3); + INSERT INTO sch1.tab1 VALUES (1, 2, 3); + CREATE PUBLICATION tap_pub_col FOR TABLE tab1 EXCEPT (a), sch1.tab1 EXCEPT (b, c); +)); + +$node_subscriber->safe_psql( + 'postgres', qq ( + CREATE SCHEMA sch1; + CREATE TABLE tab1 (a int, b int NOT NULL, c int); + CREATE TABLE sch1.tab1 (a int, b int, c int); + CREATE TABLE tab2 (a int, b int, c int); + CREATE TABLE tab3 (a int, bgen int, cgen int); + CREATE TABLE tab4 (a int, bgen int, cgen int); + CREATE TABLE tab5 (a int, b int, c int, d int); + CREATE TABLE tab6 (agen int, bgen int); +)); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub_col CONNECTION '$publisher_connstr' PUBLICATION tap_pub_col" +); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_col'); + +# Test initial sync +my $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM tab1"); +is($result, qq(|2|3), + 'check initial sync for EXCEPT (column-list) publication'); +$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.tab1"); +is($result, qq(1||), + 'check initial sync for EXCEPT (column-list) publication'); + +# Test incremental changes +$node_publisher->safe_psql( + 'postgres', qq ( + INSERT INTO tab1 VALUES (4, 5, 6); + INSERT INTO sch1.tab1 VALUES (4, 5, 6); +)); +$node_publisher->wait_for_catchup('tap_sub_col'); +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM tab1 ORDER BY a"); +is( $result, qq(|2|3 +|5|6), + 'check incremental insert for EXCEPT (column-list) publication'); +$result = $node_subscriber->safe_psql('postgres', + "SELECT * FROM sch1.tab1 ORDER BY a"); +is( $result, qq(1|| +4||), 'check incremental insert for EXCEPT (column-list) publication'); + +# Test for update +$node_subscriber->safe_psql( + 'postgres', qq( + CREATE UNIQUE INDEX b_idx ON tab1 (b); + ALTER TABLE tab1 REPLICA IDENTITY USING INDEX b_idx; +)); +$node_publisher->safe_psql( + 'postgres', qq( + CREATE UNIQUE INDEX b_idx ON tab1 (b); + ALTER TABLE tab1 REPLICA IDENTITY USING INDEX b_idx; + UPDATE tab1 SET a = 991, b = 992, c = 993 WHERE a = 1; +)); +$node_publisher->wait_for_catchup('tap_sub_col'); +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM tab1 ORDER BY a"); +is( $result, qq(|5|6 +|992|993), + 'check update for EXCEPT (column-list) publication'); + +# Test ALTER PUBLICATION for EXCEPT (column-list) +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_col ADD TABLE tab2 EXCEPT(b)"); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub_col REFRESH PUBLICATION"); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_col'); + +$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1, 2, 3)"); +$node_publisher->wait_for_catchup('tap_sub_col'); +$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM tab2"); +is($result, qq(1||3), 'check alter publication with EXCEPT (column-list)'); + +# Test for publication created with 'publish_generated_columns' as 'stored' on +# table 'tab3' and with column 'bgen' in column list with EXCEPT clause. +$node_publisher->safe_psql( + 'postgres', qq(INSERT INTO tab3 VALUES (1); + ALTER PUBLICATION tap_pub_col SET (publish_generated_columns = stored); + ALTER PUBLICATION tap_pub_col SET TABLE tab3 EXCEPT(bgen); +)); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub_col REFRESH PUBLICATION"); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_col'); + +$node_publisher->safe_psql('postgres', "INSERT INTO tab3 VALUES (2)"); +$node_publisher->wait_for_catchup('tap_sub_col'); +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM tab3 ORDER BY a"); + +# column 'bgen' is specified in EXCEPT (columm-list). So data corresponding to +# 'bgen' is not replicated. Parameter 'publish_generated_columns' is set as +# 'stored', so data corresponding to column 'cgen' is replicated. +is( $result, qq(1||3 +2||6), + 'check publication(publish_generated_columns as stored) with generated columns and EXCEPT (column-list)' +); + +# Test for publication created with 'publish_generated_columns' as 'none' on +# table with generated columns and column list specified with EXCEPT clause. +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO tab4 VALUES (1); + ALTER PUBLICATION tap_pub_col SET (publish_generated_columns = none); + ALTER PUBLICATION tap_pub_col SET TABLE tab4 EXCEPT(bgen); +)); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub_col REFRESH PUBLICATION"); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_col'); + +$node_publisher->safe_psql('postgres', "INSERT INTO tab4 VALUES (2)"); +$node_publisher->wait_for_catchup('tap_sub_col'); +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM tab4 ORDER BY a"); + +# column 'bgen' is specified in EXCEPT (columm-list). So data corresponding to +# 'bgen' is not replicated. Parameter 'publish_generated_columns' is set as +# 'none', so data corresponding to column 'cgen' is not replicated. +is( $result, qq(1|| +2||), + 'check publication(publish_generated_columns as none) with generated columns and EXCEPT (column-list)' +); + +# All columns are present in EXCEPT (column-list) +$node_publisher->safe_psql( + 'postgres', qq( + ALTER PUBLICATION tap_pub_col SET TABLE tab5 EXCEPT(a, b, c); + INSERT INTO tab5 VALUES (1, 2, 3); +)); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub_col REFRESH PUBLICATION"); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_col'); + +$node_publisher->safe_psql('postgres', "INSERT INTO tab5 VALUES (4, 5, 6)"); +$node_publisher->wait_for_catchup('tap_sub_col'); +$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5"); +is($result, qq(), 'all columns are specified in EXCEPT (column-list)'); + +# Add a new column and check that it is replicated +$node_publisher->safe_psql( + 'postgres', qq( + ALTER TABLE tab5 ADD COLUMN d int; +)); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab5 VALUES (7, 8, 9, 10)"); +$node_publisher->wait_for_catchup('tap_sub_col'); + +$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5"); +is($result, qq(|||10), 'newly added column are is replicated'); + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); + +done_testing(); -- 2.34.1