From c0cce1fc702d273d667a4356e2059837fc152b44 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 17 Aug 2023 12:56:01 -0400 Subject: [PATCH 9/9] Add TAP tests (this is broken, doesn't work). --- src/bin/pg_combinebackup/Makefile | 6 + src/bin/pg_combinebackup/meson.build | 8 +- src/bin/pg_combinebackup/t/001_basic.pl | 23 ++ .../pg_combinebackup/t/002_compare_backups.pl | 276 ++++++++++++++++++ src/test/perl/PostgreSQL/Test/Cluster.pm | 21 +- 5 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 src/bin/pg_combinebackup/t/001_basic.pl create mode 100644 src/bin/pg_combinebackup/t/002_compare_backups.pl diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile index cb20480aae..78ba05e624 100644 --- a/src/bin/pg_combinebackup/Makefile +++ b/src/bin/pg_combinebackup/Makefile @@ -44,3 +44,9 @@ uninstall: clean distclean maintainer-clean: rm -f pg_combinebackup$(X) $(OBJS) + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index bea0db405e..a6036dea74 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -25,5 +25,11 @@ bin_targets += pg_combinebackup tests += { 'name': 'pg_combinebackup', 'sd': meson.current_source_dir(), - 'bd': meson.current_build_dir() + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_basic.pl', + 't/002_compare_backups.pl', + ], + } } diff --git a/src/bin/pg_combinebackup/t/001_basic.pl b/src/bin/pg_combinebackup/t/001_basic.pl new file mode 100644 index 0000000000..fb66075d1a --- /dev/null +++ b/src/bin/pg_combinebackup/t/001_basic.pl @@ -0,0 +1,23 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Utils; +use Test::More; + +my $tempdir = PostgreSQL::Test::Utils::tempdir; + +program_help_ok('pg_combinebackup'); +program_version_ok('pg_combinebackup'); +program_options_handling_ok('pg_combinebackup'); + +command_fails_like( + ['pg_combinebackup'], + qr/no input directories specified/, + 'input directories must be specified'); +command_fails_like( + [ 'pg_combinebackup', $tempdir ], + qr/no output directory specified/, + 'output directory must be specified'); + +done_testing(); diff --git a/src/bin/pg_combinebackup/t/002_compare_backups.pl b/src/bin/pg_combinebackup/t/002_compare_backups.pl new file mode 100644 index 0000000000..c27f999a32 --- /dev/null +++ b/src/bin/pg_combinebackup/t/002_compare_backups.pl @@ -0,0 +1,276 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Set up a new database instance. +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(has_archiving => 1, allows_streaming => 1); +$primary->append_conf('postgresql.conf', 'autovacuum = off'); +$primary->start; + +# Create some test tables, each containing one row of data, plus a whole +# extra database. +$primary->safe_psql('postgres', <backup_dir . '/backup1'; +$primary->command_ok( + [ 'pg_basebackup', '-D', $backup1path, '--no-sync', '-cfast' ], + "full backup"); + +# Now make some database changes. +$primary->safe_psql('postgres', <backup_dir . '/backup2'; +$primary->command_ok( + [ 'pg_basebackup', '-D', $backup2path, '--no-sync', '-cfast', + '--incremental', $backup1path . '/backup_manifest' ], + "incremental backup"); + +# Find an LSN to which either backup can be recovered. +my $lsn = $primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); + +# Make sure that the WAL segment containing that LSN has been archived. +# PostgreSQL won't issue two consecutive XLOG_SWITCH records, and the backup +# just issued one, so call txid_current() to generate some WAL activity +# before calling pg_switch_wal(). +$primary->safe_psql('postgres', 'SELECT txid_current();'); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); + +# Now wait for the LSN we chose above to be archived. +my $archive_wait_query = + "SELECT pg_walfile_name('$lsn') <= last_archived_wal FROM pg_stat_archiver;"; +$primary->poll_query_until('postgres', $archive_wait_query) + or die "Timed out while waiting for WAL segment to be archived"; + +# Perform PITR from the full backup. Disable archive_mode so that the archive +# doesn't find out about the new timeline; that way, the later PITR below will +# choose the same timeline. +my $pitr1 = PostgreSQL::Test::Cluster->new('pitr1'); +$pitr1->init_from_backup($primary, 'backup1', + standby => 1, has_restoring => 1); +$pitr1->append_conf('postgresql.conf', qq{ +recovery_target_lsn = '$lsn' +recovery_target_action = 'promote' +archive_mode = 'off' +}); +$pitr1->start(); + +# Wait until we exit recovery, then stop the server. +$pitr1->poll_query_until('postgres', + "SELECT NOT pg_is_in_recovery();") + or die "Timed out while waiting apply to reach LSN $lsn"; +$pitr1->stop; + +# Perform PITR to the same LSN from the incremental backup. Use the same +# basic configuration as before. +my $pitr2 = PostgreSQL::Test::Cluster->new('pitr2'); +$pitr2->init_from_backup($primary, 'backup2', + standby => 1, has_restoring => 1, + combine_with_prior => [ 'backup1' ]); +$pitr2->append_conf('postgresql.conf', qq{ +recovery_target_lsn = '$lsn' +recovery_target_action = 'promote' +archive_mode = 'off' +}); +$pitr2->start(); + +# Wait until we exit recovery, then stop the server. +$pitr2->poll_query_until('postgres', + "SELECT NOT pg_is_in_recovery();") + or die "Timed out while waiting apply to reach LSN $lsn"; +$pitr2->stop; + +my $cmp = compare_data_directories($pitr1->basedir . '/pgdata', + $pitr2->basedir . '/pgdata', ''); +is($cmp, 0, "directories are identical"); + +done_testing(); + +sub compare_data_directories +{ + my ($basedir1, $basedir2, $relpath) = @_; + my $result = 0; + + if ($relpath eq '/pg_wal') + { + # Since recovery started at different LSNs, pg_wal contents may not + # be identical. Ignore that. + return 0; + } + + my $dir1 = $basedir1 . $relpath; + my $dir2 = $basedir2 . $relpath; + + opendir(DIR1, $dir1) || die "$dir1: $!"; + my @files1 = grep { $_ ne '.' && $_ ne '..' } readdir(DIR1); + closedir(DIR1); + + opendir(DIR2, $dir2) || die "$dir2: $!"; + my %files2 = map { $_ => 'unmatched' } + grep { $_ ne '.' && $_ ne '..' } readdir(DIR2); + closedir(DIR2); + + for my $fname (@files1) + { + if (!exists $files2{$fname}) + { + warn "$dir1/$fname exists but $dir2/$fname does not"; + ++$result; + next; + } + + $files2{$fname} = 'matched'; + + if (-d "$dir1/$fname") + { + if (! -d "$dir2/$fname") + { + warn "$dir1/$fname is a directory but $dir2/$fname is not"; + ++$result; + } + else + { + $result += + compare_data_directories($basedir1, $basedir2, + "$relpath/$fname"); + } + } + elsif (-d "$dir2/$fname") + { + warn "$dir2/$fname is a directory but $dir1/$fname is not"; + ++$result; + } + else + { + # Both are plain files. + $result += compare_files($basedir1, $basedir2, "$relpath/$fname"); + } + } + + for my $fname (keys %files2) + { + if ($files2{$fname} eq 'unmatched') + { + warn "$dir2/$fname exists but $dir1/$fname does not"; + ++$result; + } + } + + return $result; +} + +sub compare_files +{ + my ($basedir1, $basedir2, $relpath) = @_; + my $file1 = $basedir1 . $relpath; + my $file2 = $basedir2 . $relpath; + + if ($relpath eq '/backup_manifest') + { + # We don't expect the backup manifest to be identical between two + # backups taken at different times, so just disregard it. + return 0; + } + + if ($relpath eq '/backup_label.old') + { + # We don't expect the backup label to be identical; the start WAL + # location and probably also the start time are expected to be + # different. + return 0; + } + + if ($relpath eq '/postgresql.conf') + { + # At least the port numbers are expected to be different, so + # disregard this file. + return 0; + } + + if ($relpath eq '/postmaster.opts') + { + # At least the cluster names are expected to be different, so + # disregard this file. + return 0; + } + + if ($relpath eq '/global/pg_control') + { + # At least the mock authentication nonce is expected to be different, + # so disregard this file. + return 0; + } + + if ($relpath eq '/pg_stat/pgstat.stat') + { + # Stats aren't stable enough to be compared here. + return 0; + } + + if ($relpath =~ m@/pg_internal\.init$@) + { + # relcache init files are rebuilt at startup, so they don't need to + # match. And because we write out the contents of data structures like + # RelationData that include pointers, they almost certainly won't. + return 0; + } + + # Check whether the lengths match. + my $length1 = -s $file1; + my $length2 = -s $file2; + if ($length1 != $length2) + { + warn "$file1 has length $length1, but $file2 has length $length2"; + return 1; + } + + # Compare contents. + my $contents1 = slurp_file($file1); + my $contents2 = slurp_file($file2); + if ($contents1 ne $contents2) + { + my $nchars = 1; + while (substr($contents1, 0, $nchars) eq substr($contents2, 0, $nchars)) + { + ++$nchars; + } + warn sprintf("%s and %s are both of length %s, but differ beginning at byte %d", + $file1, $file2, $length1, $nchars - 1); + return 1; + } + + # Files are identical. + return 0; +} diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm index 2a478ba6ed..3b57379e13 100644 --- a/src/test/perl/PostgreSQL/Test/Cluster.pm +++ b/src/test/perl/PostgreSQL/Test/Cluster.pm @@ -779,6 +779,10 @@ a tar-format backup, pass the name of the tar program to use in the keyword parameter tar_program. Note that tablespace tar files aren't handled here. +To restore from an incremental backup, pass the parameter combine_with_prior +as a reference to an array of prior backup names with which this backup +is to be combined using pg_combinebackup. + Streaming replication can be enabled on this node by passing the keyword parameter has_streaming => 1. This is disabled by default. @@ -816,7 +820,22 @@ sub init_from_backup mkdir $self->archive_dir; my $data_path = $self->data_dir; - if (defined $params{tar_program}) + if (defined $params{combine_with_prior}) + { + my @prior_backups = @{$params{combine_with_prior}}; + my @prior_backup_path; + + for my $prior_backup_name (@prior_backups) + { + push @prior_backup_path, + $root_node->backup_dir . '/' . $prior_backup_name; + } + + local %ENV = $self->_get_env(); + PostgreSQL::Test::Utils::system_or_bail('pg_combinebackup', + @prior_backup_path, $backup_path, '-o', $data_path); + } + elsif (defined $params{tar_program}) { mkdir($data_path); PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf', -- 2.37.1 (Apple Git-137.1)