From 0fd6b72e710e08e96230e9b09974044eccdd4d44 Mon Sep 17 00:00:00 2001 From: Amul Sul Date: Tue, 2 Jul 2024 17:04:56 +0530 Subject: [PATCH v2 10/10] pg_verifybackup: Tests and document ---- NOTE: This patch is not meant to be committed separately. It should be squashed with the previous patch that implements tar format support. ---- --- doc/src/sgml/ref/pg_verifybackup.sgml | 54 +++++++++++++- src/bin/pg_verifybackup/t/001_basic.pl | 18 ++++- src/bin/pg_verifybackup/t/004_options.pl | 2 +- src/bin/pg_verifybackup/t/005_bad_manifest.pl | 2 +- src/bin/pg_verifybackup/t/008_untar.pl | 73 ++++++------------- src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +----------- 6 files changed, 96 insertions(+), 101 deletions(-) diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml index a3f167f9f6e..c743bd89a92 100644 --- a/doc/src/sgml/ref/pg_verifybackup.sgml +++ b/doc/src/sgml/ref/pg_verifybackup.sgml @@ -34,8 +34,10 @@ PostgreSQL documentation integrity of a database cluster backup taken using pg_basebackup against a backup_manifest generated by the server at the time - of the backup. The backup must be stored in the "plain" - format; a "tar" format backup can be checked after extracting it. + of the backup. The backup must be stored in the "plain" or "tar" + format. Verification is supported for gzip, + lz4, and zstd compressed tar backup; + any other compressed format backups can be checked after decompressing them. @@ -168,6 +170,42 @@ PostgreSQL documentation + + + + + + Specifies the format of the backup. format + can be one of the following: + + + + p + plain + + + Backup consists of plain files with the same layout as the + source server's data directory and tablespaces. + + + + + + t + tar + + + Backup consists of tar files. The main data directory's contents + will be written to a file named base.tar, + and each other tablespace will be written to a separate tar file + named after that tablespace's OID. + + + + + + + @@ -227,6 +265,18 @@ PostgreSQL documentation + + + + + + + The tar backup compression method can be gzip, + lz4, zstd, or + none if no compression. + + + diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl index 2f3e52d296f..d47ce1f04fc 100644 --- a/src/bin/pg_verifybackup/t/001_basic.pl +++ b/src/bin/pg_verifybackup/t/001_basic.pl @@ -17,13 +17,25 @@ command_fails_like( qr/no backup directory specified/, 'target directory must be specified'); command_fails_like( - [ 'pg_verifybackup', $tempdir ], + [ 'pg_verifybackup', '-Fp', $tempdir ], qr/could not open file.*\/backup_manifest\"/, 'pg_verifybackup requires a manifest'); command_fails_like( - [ 'pg_verifybackup', $tempdir, $tempdir ], + [ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ], qr/too many command-line arguments/, 'multiple target directories not allowed'); +command_fails_like( + [ 'pg_verifybackup', '-Zgzip', $tempdir ], + qr/only tar mode backups can be compressed/, + 'compression method required format option'); +command_fails_like( + [ 'pg_verifybackup', '-Fp', '-Zlz4', $tempdir ], + qr/only tar mode backups can be compressed/, + 'compression method required tar format option'); +command_fails_like( + [ 'pg_verifybackup', '-Fp', '-Znon_exist', $tempdir ], + qr/unrecognized compression algorithm/, + 'compression method should be valid'); # create fake manifest file open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!"; @@ -31,7 +43,7 @@ close($fh); # but then try to use an alternate, nonexisting manifest command_fails_like( - [ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ], + [ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ], qr/could not open file.*\/not_the_manifest\"/, 'pg_verifybackup respects -m flag'); diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl index 8ed2214408e..2f197648740 100644 --- a/src/bin/pg_verifybackup/t/004_options.pl +++ b/src/bin/pg_verifybackup/t/004_options.pl @@ -108,7 +108,7 @@ unlike( # Test valid manifest with nonexistent backup directory. command_fails_like( [ - 'pg_verifybackup', '-m', + 'pg_verifybackup', '-Fp', '-m', "$backup_path/backup_manifest", "$backup_path/fake" ], qr/could not open directory/, diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl index c4ed64b62d5..28c51b6feb0 100644 --- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl +++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl @@ -208,7 +208,7 @@ sub test_bad_manifest print $fh $manifest_contents; close($fh); - command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name); + command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name); return; } diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl index 7a09f3b75b2..9896560adc3 100644 --- a/src/bin/pg_verifybackup/t/008_untar.pl +++ b/src/bin/pg_verifybackup/t/008_untar.pl @@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary'); $primary->init(allows_streaming => 1); $primary->start; +# Create a couple of directories to use as tablespaces. +my $TS1_LOCATION = $primary->backup_dir .'/ts1'; +$TS1_LOCATION =~ s/\/\.\//\//g; # collapse foo/./bar to foo/bar +mkdir($TS1_LOCATION); + +# Create a tablespace with table in it. +$primary->safe_psql('postgres', qq( + CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION'; + SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'; + CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1; + INSERT INTO regress_tbl1 VALUES(generate_series(1,5));)); +my $tsoid = $primary->safe_psql('postgres', qq( + SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1')); + my $backup_path = $primary->backup_dir . '/server-backup'; my $extract_path = $primary->backup_dir . '/extracted-backup'; @@ -23,39 +37,31 @@ my @test_configuration = ( { 'compression_method' => 'none', 'backup_flags' => [], - 'backup_archive' => 'base.tar', + 'backup_archive' => ['base.tar', "$tsoid.tar"], 'enabled' => 1 }, { 'compression_method' => 'gzip', 'backup_flags' => [ '--compress', 'server-gzip' ], - 'backup_archive' => 'base.tar.gz', - 'decompress_program' => $ENV{'GZIP_PROGRAM'}, - 'decompress_flags' => ['-d'], + 'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ], 'enabled' => check_pg_config("#define HAVE_LIBZ 1") }, { 'compression_method' => 'lz4', 'backup_flags' => [ '--compress', 'server-lz4' ], - 'backup_archive' => 'base.tar.lz4', - 'decompress_program' => $ENV{'LZ4'}, - 'decompress_flags' => [ '-d', '-m' ], + 'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ], 'enabled' => check_pg_config("#define USE_LZ4 1") }, { 'compression_method' => 'zstd', 'backup_flags' => [ '--compress', 'server-zstd' ], - 'backup_archive' => 'base.tar.zst', - 'decompress_program' => $ENV{'ZSTD'}, - 'decompress_flags' => ['-d'], + 'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ], 'enabled' => check_pg_config("#define USE_ZSTD 1") }, { 'compression_method' => 'zstd', 'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ], - 'backup_archive' => 'base.tar.zst', - 'decompress_program' => $ENV{'ZSTD'}, - 'decompress_flags' => ['-d'], + 'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ], 'enabled' => check_pg_config("#define USE_ZSTD 1") }); @@ -86,47 +92,16 @@ for my $tc (@test_configuration) my $backup_files = join(',', sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path)); my $expected_backup_files = - join(',', sort ('backup_manifest', $tc->{'backup_archive'})); + join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} })); is($backup_files, $expected_backup_files, "found expected backup files, compression $method"); - # Decompress. - if (exists $tc->{'decompress_program'}) - { - my @decompress = ($tc->{'decompress_program'}); - push @decompress, @{ $tc->{'decompress_flags'} } - if $tc->{'decompress_flags'}; - push @decompress, $backup_path . '/' . $tc->{'backup_archive'}; - system_or_bail(@decompress); - } - - SKIP: - { - my $tar = $ENV{TAR}; - # don't check for a working tar here, to accommodate various odd - # cases. If tar doesn't work the init_from_backup below will fail. - skip "no tar program available", 1 - if (!defined $tar || $tar eq ''); - - # Untar. - mkdir($extract_path); - system_or_bail($tar, 'xf', $backup_path . '/base.tar', - '-C', $extract_path); - - # Verify. - $primary->command_ok( - [ - 'pg_verifybackup', '-n', - '-m', "$backup_path/backup_manifest", - '-e', $extract_path - ], - "verify backup, compression $method"); - } + # Verify tar backup. + $primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path], + "verify backup, compression $method"); # Cleanup. - unlink($backup_path . '/backup_manifest'); - unlink($backup_path . '/base.tar'); - unlink($backup_path . '/' . $tc->{'backup_archive'}); + rmtree($backup_path); rmtree($extract_path); } } diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl index 8c076d46dee..6b7d7483f6e 100644 --- a/src/bin/pg_verifybackup/t/010_client_untar.pl +++ b/src/bin/pg_verifybackup/t/010_client_untar.pl @@ -29,41 +29,30 @@ my @test_configuration = ( 'compression_method' => 'gzip', 'backup_flags' => [ '--compress', 'client-gzip:5' ], 'backup_archive' => 'base.tar.gz', - 'decompress_program' => $ENV{'GZIP_PROGRAM'}, - 'decompress_flags' => ['-d'], 'enabled' => check_pg_config("#define HAVE_LIBZ 1") }, { 'compression_method' => 'lz4', 'backup_flags' => [ '--compress', 'client-lz4:5' ], 'backup_archive' => 'base.tar.lz4', - 'decompress_program' => $ENV{'LZ4'}, - 'decompress_flags' => ['-d'], - 'output_file' => 'base.tar', 'enabled' => check_pg_config("#define USE_LZ4 1") }, { 'compression_method' => 'zstd', 'backup_flags' => [ '--compress', 'client-zstd:5' ], 'backup_archive' => 'base.tar.zst', - 'decompress_program' => $ENV{'ZSTD'}, - 'decompress_flags' => ['-d'], 'enabled' => check_pg_config("#define USE_ZSTD 1") }, { 'compression_method' => 'zstd', 'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ], 'backup_archive' => 'base.tar.zst', - 'decompress_program' => $ENV{'ZSTD'}, - 'decompress_flags' => ['-d'], 'enabled' => check_pg_config("#define USE_ZSTD 1") }, { 'compression_method' => 'parallel zstd', 'backup_flags' => [ '--compress', 'client-zstd:workers=3' ], 'backup_archive' => 'base.tar.zst', - 'decompress_program' => $ENV{'ZSTD'}, - 'decompress_flags' => ['-d'], 'enabled' => check_pg_config("#define USE_ZSTD 1"), 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/ @@ -118,40 +107,9 @@ for my $tc (@test_configuration) is($backup_files, $expected_backup_files, "found expected backup files, compression $method"); - # Decompress. - if (exists $tc->{'decompress_program'}) - { - my @decompress = ($tc->{'decompress_program'}); - push @decompress, @{ $tc->{'decompress_flags'} } - if $tc->{'decompress_flags'}; - push @decompress, $backup_path . '/' . $tc->{'backup_archive'}; - push @decompress, $backup_path . '/' . $tc->{'output_file'} - if $tc->{'output_file'}; - system_or_bail(@decompress); - } - - SKIP: - { - my $tar = $ENV{TAR}; - # don't check for a working tar here, to accommodate various odd - # cases. If tar doesn't work the init_from_backup below will fail. - skip "no tar program available", 1 - if (!defined $tar || $tar eq ''); - - # Untar. - mkdir($extract_path); - system_or_bail($tar, 'xf', $backup_path . '/base.tar', - '-C', $extract_path); - - # Verify. - $primary->command_ok( - [ - 'pg_verifybackup', '-n', - '-m', "$backup_path/backup_manifest", - '-e', $extract_path - ], - "verify backup, compression $method"); - } + # Verify tar backup. + $primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ], + "verify backup, compression $method"); # Cleanup. rmtree($extract_path); -- 2.18.0