From 82fa4302fd7e2319bbdb89049a702b5eb02d12ee Mon Sep 17 00:00:00 2001 From: Khanna Date: Fri, 28 Feb 2025 17:29:30 +0530 Subject: [PATCH v13] Enhance 'pg_createsubscriber' to fetch and append all databases This patch enhances the 'pg_createsubscriber' utility by adding the '--all' option. When '--all' is specified, the tool queries the source server (publisher) for all databases and creates subscriptions on the target server (subscriber) for databases with matching names. This simplifies the process of converting a physical standby to a logical subscriber, particularly during upgrades. The options '--database', '--publication', '--subscription', and '--replication-slot' cannot be used when '--all' is specified. --- doc/src/sgml/ref/pg_createsubscriber.sgml | 52 ++- src/bin/pg_basebackup/pg_createsubscriber.c | 92 ++++- .../t/040_pg_createsubscriber.pl | 373 ----------------- .../t/041_covert_standby_to_subscriber.pl | 386 ++++++++++++++++++ src/bin/pg_basebackup/t/042_all_option.pl | 254 ++++++++++++ 5 files changed, 774 insertions(+), 383 deletions(-) create mode 100644 src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl create mode 100644 src/bin/pg_basebackup/t/042_all_option.pl diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml index b4b996236e4..7376c61f3eb 100644 --- a/doc/src/sgml/ref/pg_createsubscriber.sgml +++ b/doc/src/sgml/ref/pg_createsubscriber.sgml @@ -20,6 +20,27 @@ PostgreSQL documentation + + pg_createsubscriber + option + + + + + + + + + + datadir + + + + + connstr + + + pg_createsubscriber option @@ -87,6 +108,21 @@ PostgreSQL documentation command-line arguments: + + + + + + For all source server non-template databases create subscriptions for + databases with the same names on the target server. + Subscription names, publication names, and replication slot names are + automatically generated. This option cannot be used together with + , , + or . + + + + @@ -94,9 +130,10 @@ PostgreSQL documentation The name of the database in which to create a subscription. Multiple databases can be selected by writing multiple - switches. If option is not provided, the database - name will be obtained from option. If the database - name is not specified in either the option or + switches. This option cannot be used together with . + If option is not provided, the database name will be + obtained from option. If the database name is not + specified in either the option or option, an error will be reported. @@ -230,7 +267,8 @@ PostgreSQL documentation names must match the number of specified databases, otherwise an error is reported. The order of the multiple publication name switches must match the order of database switches. If this option is not specified, - a generated name is assigned to the publication name. + a generated name is assigned to the publication name. This option cannot + be used together with . @@ -246,7 +284,8 @@ PostgreSQL documentation otherwise an error is reported. The order of the multiple replication slot name switches must match the order of database switches. If this option is not specified, the subscription name is assigned to the - replication slot name. + replication slot name. This option cannot be used together with + . @@ -261,7 +300,8 @@ PostgreSQL documentation names must match the number of specified databases, otherwise an error is reported. The order of the multiple subscription name switches must match the order of database switches. If this option is not specified, - a generated name is assigned to the subscription name. + a generated name is assigned to the subscription name. This option cannot + be used together with . diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c index 6baf92e8024..f918a0e33b8 100644 --- a/src/bin/pg_basebackup/pg_createsubscriber.c +++ b/src/bin/pg_basebackup/pg_createsubscriber.c @@ -44,6 +44,7 @@ struct CreateSubscriberOptions SimpleStringList sub_names; /* list of subscription names */ SimpleStringList replslot_names; /* list of replication slot names */ int recovery_timeout; /* stop recovery after this time */ + bool all_dbs; /* --all option was specified */ }; /* per-database publication/subscription info */ @@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn, const struct LogicalRepInfo *dbinfo); static void drop_existing_subscriptions(PGconn *conn, const char *subname, const char *dbname); +static void fetch_source_databases(struct CreateSubscriberOptions *opt); #define USEC_PER_SEC 1000000 #define WAIT_INTERVAL 1 /* 1 second */ @@ -236,6 +238,7 @@ usage(void) printf(_("Usage:\n")); printf(_(" %s [OPTION]...\n"), progname); printf(_("\nOptions:\n")); + printf(_(" -a, --all create subscriptions for all non-template source databases\n")); printf(_(" -d, --database=DBNAME database in which to create a subscription\n")); printf(_(" -D, --pgdata=DATADIR location for the subscriber data directory\n")); printf(_(" -n, --dry-run dry run, just show what would be done\n")); @@ -1904,11 +1907,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo) destroyPQExpBuffer(str); } +/* + * If --all is specified, fetch a list of all user-created databases from the + * source server. Internally, this is treated as if the user specified multiple + * --database options, one for each source database. + */ +static void +fetch_source_databases(struct CreateSubscriberOptions *opt) +{ + PGconn *conn; + PGresult *res; + + /* Establish a connection to the source server */ + conn = connect_database(opt->pub_conninfo_str, true); + + res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true"); + + /* Check for errors during query execution */ + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res)); + PQclear(res); + disconnect_database(conn, true); + } + + /* Process the query result */ + for (int i = 0; i < PQntuples(res); i++) + { + const char *dbname = PQgetvalue(res, i, 0); + + simple_string_list_append(&opt->database_names, dbname); + + /* Increment num_dbs to reflect multiple --database options */ + num_dbs++; + } + + /* Error if no databases were found on the source server */ + if (num_dbs == 0) + { + pg_log_error("no suitable databases found on the source server"); + pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server."); + PQclear(res); + disconnect_database(conn, true); + } + + PQclear(res); + disconnect_database(conn, false); +} + int main(int argc, char **argv) { static struct option long_options[] = { + {"all", no_argument, NULL, 'a'}, {"database", required_argument, NULL, 'd'}, {"pgdata", required_argument, NULL, 'D'}, {"dry-run", no_argument, NULL, 'n'}, @@ -1978,6 +2030,7 @@ main(int argc, char **argv) 0 }; opt.recovery_timeout = 0; + opt.all_dbs = false; /* * Don't allow it to be run as root. It uses pg_ctl which does not allow @@ -1995,11 +2048,14 @@ main(int argc, char **argv) get_restricted_token(); - while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v", + while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v", long_options, &option_index)) != -1) { switch (c) { + case 'a': + opt.all_dbs = true; + break; case 'd': if (!simple_string_list_member(&opt.database_names, optarg)) { @@ -2087,6 +2143,28 @@ main(int argc, char **argv) } } + /* Validate that --all is not used with incompatible options */ + if (opt.all_dbs) + { + char *bad_switch = NULL; + + if (num_dbs > 0) + bad_switch = "--database"; + else if (num_pubs > 0) + bad_switch = "--publication"; + else if (num_replslots > 0) + bad_switch = "--replication-slot"; + else if (num_subs > 0) + bad_switch = "--subscription"; + + if (bad_switch) + { + pg_log_error("%s cannot be used with --all", bad_switch); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + } + /* Any non-option arguments? */ if (optind < argc) { @@ -2140,14 +2218,20 @@ main(int argc, char **argv) pg_log_info("validating subscriber connection string"); sub_base_conninfo = get_sub_conninfo(&opt); + /* + * Fetch all databases from the source (publisher) if --all is specified. + */ + if (opt.all_dbs) + fetch_source_databases(&opt); + if (opt.database_names.head == NULL) { pg_log_info("no database was specified"); /* - * If --database option is not provided, try to obtain the dbname from - * the publisher conninfo. If dbname parameter is not available, error - * out. + * If neither --database nor --all option is provided, try to obtain + * the dbname from the publisher conninfo. If dbname parameter is not + * available, error out. */ if (dbname_conninfo) { diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl index c35fa108ce3..d6e97405ee1 100644 --- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl +++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl @@ -15,32 +15,6 @@ program_options_handling_ok('pg_createsubscriber'); my $datadir = PostgreSQL::Test::Utils::tempdir; -# Generate a database with a name made of a range of ASCII characters. -# Extracted from 002_pg_upgrade.pl. -sub generate_db -{ - my ($node, $prefix, $from_char, $to_char, $suffix) = @_; - - my $dbname = $prefix; - for my $i ($from_char .. $to_char) - { - next if $i == 7 || $i == 10 || $i == 13; # skip BEL, LF, and CR - $dbname = $dbname . sprintf('%c', $i); - } - - $dbname .= $suffix; - - # On Windows, older IPC::Run versions can mis-quote command line arguments - # containing double quote or backslash - $dbname =~ tr/\"\\//d if ($windows_os); - - $node->command_ok( - [ 'createdb', $dbname ], - "created database with ASCII characters from $from_char to $to_char"); - - return $dbname; -} - # # Test mandatory options command_fails(['pg_createsubscriber'], @@ -118,351 +92,4 @@ command_fails( ], 'wrong number of replication slot names'); -# Set up node P as primary -my $node_p = PostgreSQL::Test::Cluster->new('node_p'); -my $pconnstr = $node_p->connstr; -$node_p->init(allows_streaming => 'logical'); -# Disable autovacuum to avoid generating xid during stats update as otherwise -# the new XID could then be replicated to standby at some random point making -# slots at primary lag behind standby during slot sync. -$node_p->append_conf('postgresql.conf', 'autovacuum = off'); -$node_p->start; - -# Set up node F as about-to-fail node -# Force it to initialize a new cluster instead of copying a -# previously initdb'd cluster. New cluster has a different system identifier so -# we can test if the target cluster is a copy of the source cluster. -my $node_f = PostgreSQL::Test::Cluster->new('node_f'); -$node_f->init(force_initdb => 1, allows_streaming => 'logical'); - -# On node P -# - create databases -# - create test tables -# - insert a row -# - create a physical replication slot -my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\'); -my $db2 = generate_db($node_p, 'regression', 46, 90, ''); - -$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)'); -$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')"); -$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)'); -my $slotname = 'physical_slot'; -$node_p->safe_psql($db2, - "SELECT pg_create_physical_replication_slot('$slotname')"); - -# Set up node S as standby linking to node P -$node_p->backup('backup_1'); -my $node_s = PostgreSQL::Test::Cluster->new('node_s'); -$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1); -$node_s->append_conf( - 'postgresql.conf', qq[ -primary_slot_name = '$slotname' -primary_conninfo = '$pconnstr dbname=postgres' -hot_standby_feedback = on -]); -$node_s->set_standby_mode(); -$node_s->start; - -# Set up node T as standby linking to node P then promote it -my $node_t = PostgreSQL::Test::Cluster->new('node_t'); -$node_t->init_from_backup($node_p, 'backup_1', has_streaming => 1); -$node_t->set_standby_mode(); -$node_t->start; -$node_t->promote; -$node_t->stop; - -# Run pg_createsubscriber on a promoted server -command_fails( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--pgdata' => $node_t->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_t->host, - '--subscriber-port' => $node_t->port, - '--database' => $db1, - '--database' => $db2, - ], - 'target server is not in recovery'); - -# Run pg_createsubscriber when standby is running -command_fails( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--pgdata' => $node_s->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_s->host, - '--subscriber-port' => $node_s->port, - '--database' => $db1, - '--database' => $db2, - ], - 'standby is up and running'); - -# Run pg_createsubscriber on about-to-fail node F -command_fails( - [ - 'pg_createsubscriber', - '--verbose', - '--pgdata' => $node_f->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_f->host, - '--subscriber-port' => $node_f->port, - '--database' => $db1, - '--database' => $db2 - ], - 'subscriber data directory is not a copy of the source database cluster'); - -# Set up node C as standby linking to node S -$node_s->backup('backup_2'); -my $node_c = PostgreSQL::Test::Cluster->new('node_c'); -$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1); -$node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef); -$node_c->set_standby_mode(); - -# Run pg_createsubscriber on node C (P -> S -> C) -command_fails( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--pgdata' => $node_c->data_dir, - '--publisher-server' => $node_s->connstr($db1), - '--socketdir' => $node_c->host, - '--subscriber-port' => $node_c->port, - '--database' => $db1, - '--database' => $db2, - ], - 'primary server is in recovery'); - -# Check some unmet conditions on node P -$node_p->append_conf( - 'postgresql.conf', q{ -wal_level = replica -max_replication_slots = 1 -max_wal_senders = 1 -max_worker_processes = 2 -}); -$node_p->restart; -$node_s->stop; -command_fails( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--pgdata' => $node_s->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_s->host, - '--subscriber-port' => $node_s->port, - '--database' => $db1, - '--database' => $db2, - - ], - 'primary contains unmet conditions on node P'); -# Restore default settings here but only apply it after testing standby. Some -# standby settings should not be a lower setting than on the primary. -$node_p->append_conf( - 'postgresql.conf', q{ -wal_level = logical -max_replication_slots = 10 -max_wal_senders = 10 -max_worker_processes = 8 -}); - -# Check some unmet conditions on node S -$node_s->append_conf( - 'postgresql.conf', q{ -max_replication_slots = 1 -max_logical_replication_workers = 1 -max_worker_processes = 2 -}); -command_fails( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--pgdata' => $node_s->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_s->host, - '--subscriber-port' => $node_s->port, - '--database' => $db1, - '--database' => $db2, - ], - 'standby contains unmet conditions on node S'); -$node_s->append_conf( - 'postgresql.conf', q{ -max_replication_slots = 10 -max_logical_replication_workers = 4 -max_worker_processes = 8 -}); -# Restore default settings on both servers -$node_p->restart; - -# Create failover slot to test its removal -my $fslotname = 'failover_slot'; -$node_p->safe_psql($db1, - "SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)" -); -$node_s->start; -# Wait for the standby to catch up so that the standby is not lagging behind -# the failover slot. -$node_p->wait_for_replay_catchup($node_s); -$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()"); -my $result = $node_s->safe_psql('postgres', - "SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary" -); -is($result, 'failover_slot', 'failover slot is synced'); - -# Insert another row on node P and wait node S to catch up. We -# intentionally performed this insert after syncing logical slot -# as otherwise the local slot's (created during synchronization of -# slot) xmin on standby could be ahead of the remote slot leading -# to failure in synchronization. -$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')"); -$node_p->wait_for_replay_catchup($node_s); - -# Create subscription to test its removal -my $dummy_sub = 'regress_sub_dummy'; -$node_p->safe_psql($db1, - "CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)" -); -$node_p->wait_for_replay_catchup($node_s); -$node_s->stop; - -# dry run mode on node S -command_ok( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default, - '--pgdata' => $node_s->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_s->host, - '--subscriber-port' => $node_s->port, - '--publication' => 'pub1', - '--publication' => 'pub2', - '--subscription' => 'sub1', - '--subscription' => 'sub2', - '--database' => $db1, - '--database' => $db2, - ], - 'run pg_createsubscriber --dry-run on node S'); - -# Check if node S is still a standby -$node_s->start; -is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'), - 't', 'standby is in recovery'); -$node_s->stop; - -# pg_createsubscriber can run without --databases option -command_ok( - [ - 'pg_createsubscriber', - '--verbose', - '--dry-run', - '--pgdata' => $node_s->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_s->host, - '--subscriber-port' => $node_s->port, - '--replication-slot' => 'replslot1', - ], - 'run pg_createsubscriber without --databases'); - -# Run pg_createsubscriber on node S. --verbose is used twice -# to show more information. -# In passing, also test the --enable-two-phase option -command_ok( - [ - 'pg_createsubscriber', - '--verbose', '--verbose', - '--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default, - '--pgdata' => $node_s->data_dir, - '--publisher-server' => $node_p->connstr($db1), - '--socketdir' => $node_s->host, - '--subscriber-port' => $node_s->port, - '--publication' => 'pub1', - '--publication' => 'pub2', - '--replication-slot' => 'replslot1', - '--replication-slot' => 'replslot2', - '--database' => $db1, - '--database' => $db2, - '--enable-two-phase' - ], - 'run pg_createsubscriber on node S'); - -# Confirm the physical replication slot has been removed -$result = $node_p->safe_psql($db1, - "SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$slotname'" -); -is($result, qq(0), - 'the physical replication slot used as primary_slot_name has been removed' -); - -# Insert rows on P -$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')"); -$node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')"); - -# Start subscriber -$node_s->start; - -# Verify that all subtwophase states are pending or enabled, -# e.g. there are no subscriptions where subtwophase is disabled ('d') -is( $node_s->safe_psql( - 'postgres', - "SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'" - ), - 't', - 'subscriptions are created with the two-phase option enabled'); - -# Confirm the pre-existing subscription has been removed -$result = $node_s->safe_psql( - 'postgres', qq( - SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub' -)); -is($result, qq(0), 'pre-existing subscription was dropped'); - -# Get subscription names -$result = $node_s->safe_psql( - 'postgres', qq( - SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_' -)); -my @subnames = split("\n", $result); - -# Wait subscriber to catch up -$node_s->wait_for_subscription_sync($node_p, $subnames[0]); -$node_s->wait_for_subscription_sync($node_p, $subnames[1]); - -# Confirm the failover slot has been removed -$result = $node_s->safe_psql($db1, - "SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'" -); -is($result, qq(0), 'failover slot was removed'); - -# Check result in database $db1 -$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1'); -is( $result, qq(first row -second row -third row), - "logical replication works in database $db1"); - -# Check result in database $db2 -$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2'); -is($result, qq(row 1), "logical replication works in database $db2"); - -# Different system identifier? -my $sysid_p = $node_p->safe_psql('postgres', - 'SELECT system_identifier FROM pg_control_system()'); -my $sysid_s = $node_s->safe_psql('postgres', - 'SELECT system_identifier FROM pg_control_system()'); -ok($sysid_p != $sysid_s, 'system identifier was changed'); - -# clean up -$node_p->teardown_node; -$node_s->teardown_node; -$node_t->teardown_node; -$node_f->teardown_node; - done_testing(); diff --git a/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl b/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl new file mode 100644 index 00000000000..a51c86ed9f3 --- /dev/null +++ b/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl @@ -0,0 +1,386 @@ +# Copyright (c) 2024-2025, PostgreSQL Global Development Group + +# +# Test using a standby server as the subscriber. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Generate a database with a name made of a range of ASCII characters. +# Extracted from 002_pg_upgrade.pl. +sub generate_db +{ + my ($node, $prefix, $from_char, $to_char, $suffix) = @_; + + my $dbname = $prefix; + for my $i ($from_char .. $to_char) + { + next if $i == 7 || $i == 10 || $i == 13; # skip BEL, LF, and CR + $dbname = $dbname . sprintf('%c', $i); + } + + $dbname .= $suffix; + + # On Windows, older IPC::Run versions can mis-quote command line arguments + # containing double quote or backslash + $dbname =~ tr/\"\\//d if ($windows_os); + + $node->command_ok( + [ 'createdb', $dbname ], + "created database with ASCII characters from $from_char to $to_char"); + + return $dbname; +} + +# +# Set up node P as primary +my $node_p = PostgreSQL::Test::Cluster->new('node_p'); +my $pconnstr = $node_p->connstr; +$node_p->init(allows_streaming => 'logical'); +# Disable autovacuum to avoid generating xid during stats update as otherwise +# the new XID could then be replicated to standby at some random point making +# slots at primary lag behind standby during slot sync. +$node_p->append_conf('postgresql.conf', 'autovacuum = off'); +$node_p->start; + +# Set up node F as about-to-fail node +# Force it to initialize a new cluster instead of copying a +# previously initdb'd cluster. New cluster has a different system identifier so +# we can test if the target cluster is a copy of the source cluster. +my $node_f = PostgreSQL::Test::Cluster->new('node_f'); +$node_f->init(force_initdb => 1, allows_streaming => 'logical'); + +# On node P +# - create databases +# - create test tables +# - insert a row +# - create a physical replication slot +my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\'); +my $db2 = generate_db($node_p, 'regression', 46, 90, ''); + +$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)'); +$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')"); +$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)'); +my $slotname = 'physical_slot'; +$node_p->safe_psql($db2, + "SELECT pg_create_physical_replication_slot('$slotname')"); + +# Set up node S as standby linking to node P +$node_p->backup('backup_1'); +my $node_s = PostgreSQL::Test::Cluster->new('node_s'); +$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1); +$node_s->append_conf( + 'postgresql.conf', qq[ +primary_slot_name = '$slotname' +primary_conninfo = '$pconnstr dbname=postgres' +hot_standby_feedback = on +]); +$node_s->set_standby_mode(); +$node_s->start; + +# Set up node T as standby linking to node P then promote it +my $node_t = PostgreSQL::Test::Cluster->new('node_t'); +$node_t->init_from_backup($node_p, 'backup_1', has_streaming => 1); +$node_t->set_standby_mode(); +$node_t->start; +$node_t->promote; +$node_t->stop; + +# Run pg_createsubscriber on a promoted server +command_fails( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_t->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_t->host, + '--subscriber-port' => $node_t->port, + '--database' => $db1, + '--database' => $db2, + ], + 'target server is not in recovery'); + +# Run pg_createsubscriber when standby is running +command_fails( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--database' => $db1, + '--database' => $db2, + ], + 'standby is up and running'); + +# Run pg_createsubscriber on about-to-fail node F +command_fails( + [ + 'pg_createsubscriber', + '--verbose', + '--pgdata' => $node_f->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_f->host, + '--subscriber-port' => $node_f->port, + '--database' => $db1, + '--database' => $db2 + ], + 'subscriber data directory is not a copy of the source database cluster'); + +# Set up node C as standby linking to node S +$node_s->backup('backup_2'); +my $node_c = PostgreSQL::Test::Cluster->new('node_c'); +$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1); +$node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef); +$node_c->set_standby_mode(); + +# Run pg_createsubscriber on node C (P -> S -> C) +command_fails( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_c->data_dir, + '--publisher-server' => $node_s->connstr($db1), + '--socketdir' => $node_c->host, + '--subscriber-port' => $node_c->port, + '--database' => $db1, + '--database' => $db2, + ], + 'primary server is in recovery'); + +# Check some unmet conditions on node P +$node_p->append_conf( + 'postgresql.conf', q{ +wal_level = replica +max_replication_slots = 1 +max_wal_senders = 1 +max_worker_processes = 2 +}); +$node_p->restart; +$node_s->stop; +command_fails( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--database' => $db1, + '--database' => $db2, + + ], + 'primary contains unmet conditions on node P'); +# Restore default settings here but only apply it after testing standby. Some +# standby settings should not be a lower setting than on the primary. +$node_p->append_conf( + 'postgresql.conf', q{ +wal_level = logical +max_replication_slots = 10 +max_wal_senders = 10 +max_worker_processes = 8 +}); + +# Check some unmet conditions on node S +$node_s->append_conf( + 'postgresql.conf', q{ +max_replication_slots = 1 +max_logical_replication_workers = 1 +max_worker_processes = 2 +}); +command_fails( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--database' => $db1, + '--database' => $db2, + ], + 'standby contains unmet conditions on node S'); +$node_s->append_conf( + 'postgresql.conf', q{ +max_replication_slots = 10 +max_logical_replication_workers = 4 +max_worker_processes = 8 +}); +# Restore default settings on both servers +$node_p->restart; + +# Create failover slot to test its removal +my $fslotname = 'failover_slot'; +$node_p->safe_psql($db1, + "SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)" +); +$node_s->start; +# Wait for the standby to catch up so that the standby is not lagging behind +# the failover slot. +$node_p->wait_for_replay_catchup($node_s); +$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()"); +my $result = $node_s->safe_psql('postgres', + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary" +); +is($result, 'failover_slot', 'failover slot is synced'); + +# Insert another row on node P and wait node S to catch up. We +# intentionally performed this insert after syncing logical slot +# as otherwise the local slot's (created during synchronization of +# slot) xmin on standby could be ahead of the remote slot leading +# to failure in synchronization. +$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')"); +$node_p->wait_for_replay_catchup($node_s); + +# Create subscription to test its removal +my $dummy_sub = 'regress_sub_dummy'; +$node_p->safe_psql($db1, + "CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)" +); +$node_p->wait_for_replay_catchup($node_s); +$node_s->stop; + +# dry run mode on node S +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default, + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--publication' => 'pub1', + '--publication' => 'pub2', + '--subscription' => 'sub1', + '--subscription' => 'sub2', + '--database' => $db1, + '--database' => $db2, + ], + 'run pg_createsubscriber --dry-run on node S'); + +# Check if node S is still a standby +$node_s->start; +is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'), + 't', 'standby is in recovery'); +$node_s->stop; + +# pg_createsubscriber can run without --databases option +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--replication-slot' => 'replslot1', + ], + 'run pg_createsubscriber without --databases'); + +# Run pg_createsubscriber on node S. --verbose is used twice +# to show more information. +# In passing, also test the --enable-two-phase option +command_ok( + [ + 'pg_createsubscriber', + '--verbose', '--verbose', + '--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default, + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--publication' => 'pub1', + '--publication' => 'pub2', + '--replication-slot' => 'replslot1', + '--replication-slot' => 'replslot2', + '--database' => $db1, + '--database' => $db2, + '--enable-two-phase' + ], + 'run pg_createsubscriber on node S'); + +# Confirm the physical replication slot has been removed +$result = $node_p->safe_psql($db1, + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$slotname'" +); +is($result, qq(0), + 'the physical replication slot used as primary_slot_name has been removed' +); + +# Insert rows on P +$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')"); +$node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')"); + +# Start subscriber +$node_s->start; + +# Verify that all subtwophase states are pending or enabled, +# e.g. there are no subscriptions where subtwophase is disabled ('d') +is( $node_s->safe_psql( + 'postgres', + "SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'" + ), + 't', + 'subscriptions are created with the two-phase option enabled'); + +# Confirm the pre-existing subscription has been removed +$result = $node_s->safe_psql( + 'postgres', qq( + SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub' +)); +is($result, qq(0), 'pre-existing subscription was dropped'); + +# Get subscription names +$result = $node_s->safe_psql( + 'postgres', qq( + SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_' +)); +my @subnames = split("\n", $result); + +# Wait subscriber to catch up +$node_s->wait_for_subscription_sync($node_p, $subnames[0]); +$node_s->wait_for_subscription_sync($node_p, $subnames[1]); + +# Confirm the failover slot has been removed +$result = $node_s->safe_psql($db1, + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'" +); +is($result, qq(0), 'failover slot was removed'); + +# Check result in database $db1 +$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1'); +is( $result, qq(first row +second row +third row), + "logical replication works in database $db1"); + +# Check result in database $db2 +$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2'); +is($result, qq(row 1), "logical replication works in database $db2"); + +# Different system identifier? +my $sysid_p = $node_p->safe_psql('postgres', + 'SELECT system_identifier FROM pg_control_system()'); +my $sysid_s = $node_s->safe_psql('postgres', + 'SELECT system_identifier FROM pg_control_system()'); +ok($sysid_p != $sysid_s, 'system identifier was changed'); + +# clean up +$node_p->teardown_node; +$node_s->teardown_node; +$node_t->teardown_node; +$node_f->teardown_node; + +done_testing(); diff --git a/src/bin/pg_basebackup/t/042_all_option.pl b/src/bin/pg_basebackup/t/042_all_option.pl new file mode 100644 index 00000000000..18e74c294b9 --- /dev/null +++ b/src/bin/pg_basebackup/t/042_all_option.pl @@ -0,0 +1,254 @@ +# Copyright (c) 2024-2025, PostgreSQL Global Development Group + +# +# Test using a standby server as the subscriber. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Generate a database with a name made of a range of ASCII characters. +# Extracted from 002_pg_upgrade.pl. +sub generate_db +{ + my ($node, $prefix, $from_char, $to_char, $suffix) = @_; + + my $dbname = $prefix; + for my $i ($from_char .. $to_char) + { + next if $i == 7 || $i == 10 || $i == 13; # skip BEL, LF, and CR + $dbname = $dbname . sprintf('%c', $i); + } + + $dbname .= $suffix; + + # On Windows, older IPC::Run versions can mis-quote command line arguments + # containing double quote or backslash + $dbname =~ tr/\"\\//d if ($windows_os); + + $node->command_ok( + [ 'createdb', $dbname ], + "created database with ASCII characters from $from_char to $to_char"); + + return $dbname; +} + +# +# Set up node P as primary +my $node_p = PostgreSQL::Test::Cluster->new('node_p'); +my $pconnstr = $node_p->connstr; +$node_p->init(allows_streaming => 'logical'); +# Disable autovacuum to avoid generating xid during stats update as otherwise +# the new XID could then be replicated to standby at some random point making +# slots at primary lag behind standby during slot sync. +$node_p->append_conf('postgresql.conf', 'autovacuum = off'); +$node_p->start; + +# On node P +# - create databases +# - create test tables +# - insert a row +my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\'); +my $db2 = generate_db($node_p, 'regression', 46, 90, ''); + +$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)'); +$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)'); +$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')"); +$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)'); + +# Set up node S as standby linking to node P +$node_p->backup('backup_1'); +my $node_s = PostgreSQL::Test::Cluster->new('node_s'); +$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1); +$node_s->append_conf( + 'postgresql.conf', qq[ +primary_conninfo = '$pconnstr dbname=postgres' +hot_standby_feedback = on +]); +$node_s->set_standby_mode(); + +# run pg_createsubscriber with '--all' and '--database' and verify the +# failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--database' => $db1, + ], + qr/--database cannot be used with --all/, + 'fail if --database is used with --all'); + +# run pg_createsubscriber with '--all' and '--database' without '--dry-run' +# and verify the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--database' => $db1, + ], + qr/--database cannot be used with --all/, + 'fail if --database is used with --all without --dry-run'); + +# run pg_createsubscriber with '--all' and '--publication' and verify +# the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--publication' => 'pub1', + ], + qr/--publication cannot be used with --all/, + 'fail if --publication is used with --all'); + +# run pg_createsubscriber with '--all' and '--replication-slot' and +# verify the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--replication-slot' => 'replslot1', + ], + qr/--replication-slot cannot be used with --all/, + 'fail if --replication-slot is used with --all'); + +# run pg_createsubscriber with '--all' and '--subscription' and +# verify the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--subscription' => 'sub1', + ], + qr/--subscription cannot be used with --all/, + 'fail if --subscription is used with --all'); + +# run pg_createsubscriber with '--all' option +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + ], + 'run pg_createsubscriber with --all'); + +# run pg_createsubscriber with '--all' option without '--dry-run' +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + ], + 'run pg_createsubscriber with --all without --dry-run'); + +$node_s->start; + +# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both +# $db1 and $db2 must be escaped to pass the safe_psql(), but it is difficult. +# Thus, we define a cursor, obtain a dbname from the instance and compere one +# by one. +my @user_dbs = ('postgres', $db1, $db2); + +my $bgconn = $node_s->background_psql('postgres'); +$bgconn->query_safe( + qq[ + BEGIN; + DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database + WHERE subdbid = pg_database.oid and datistemplate = 'f' + ORDER BY pg_database.oid; +]); + +# Fetch from the cursor three times and confirm the existence of the +# subscription on $dbname +foreach my $dbname (@user_dbs) +{ + my $result = $bgconn->query_safe("FETCH cursor;"); + + is($result, $dbname, "subscription is created on $dbname"); +} +$bgconn->quit; + +# Verify template databases do not have subscriptions +my $result = $node_s->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_subscription, pg_database + WHERE subdbid = pg_database.oid and datistemplate = 't';" +); +is($result, '0', 'subscription is not created on template databases'); + +# Verify logical replication works for all databases +# Insert rows on node P +$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')"); +$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')"); +$node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')"); + +# Get subscription names +$result = $node_s->safe_psql( + 'postgres', qq( + SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_' +)); +my @subnames = split("\n", $result); + +# Wait subscriber to catch up +$node_s->wait_for_subscription_sync($node_p, $subnames[0]); +$node_s->wait_for_subscription_sync($node_p, $subnames[1]); +$node_s->wait_for_subscription_sync($node_p, $subnames[2]); + +# Check result in database 'postgres' of node S +$result = $node_s->safe_psql('postgres', 'SELECT * FROM tbl1'); +is($result, qq(first row), "logical replication works in database postgres"); + +# Check result in database $db1 of node S +$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl2'); +is( $result, qq(first row +second row), + "logical replication works in database $db1"); + +# Check result in database $db2 of node S +$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl3'); +is($result, qq(first row), "logical replication works in database $db2"); + +# clean up +$node_p->teardown_node; +$node_s->teardown_node; + +done_testing(); -- 2.41.0.windows.3