From 1dc5546344afaa2e3507b070da603ba9f8541483 Mon Sep 17 00:00:00 2001 From: Nitin Motiani Date: Sat, 15 Feb 2025 04:29:17 +0000 Subject: [PATCH v11 4/5] Add tests for pipe * These tests include the invalid usages of --pipe-command with other flags. * Also test pg_dump and pg_restore with pipe command along with various other flags. --- src/bin/pg_dump/t/001_basic.pl | 72 ++++++- src/bin/pg_dump/t/002_pg_dump.pl | 227 ++++++++++++++++++++ src/bin/pg_dump/t/004_pg_dump_parallel.pl | 30 +++ src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 18 ++ 4 files changed, 345 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index 509f4f9ce7d..878ebf33271 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -74,6 +74,48 @@ command_fails_like( 'pg_dump: options --statistics-only and --no-statistics cannot be used together' ); +command_fails_like( + [ 'pg_dump', '-Fd', '--pipe="cat"', '-f', 'testdir', 'test'], + qr/\Qpg_dump: error: options -f\/--file and --pipe cannot be used together\E/, + 'pg_dump: options -f/--file and --pipe cannot be used together' +); + +command_fails_like( + [ 'pg_dump', '-Fd', '--pipe="cat"', '-Z', 'gzip', 'test'], + qr/\Qpg_dump: error: option --pipe is not supported with any compression type\E/, + 'pg_dump: option --pipe is not supported with any compression type' +); + +command_fails_like( + [ 'pg_dump', '-Fd', '--pipe="cat"', '--compress=lz4', 'test'], + qr/\Qpg_dump: error: option --pipe is not supported with any compression type\E/, + 'pg_dump: option --pipe is not supported with any compression type' +); + +command_fails_like( + [ 'pg_dump', '-Fd', '--pipe="cat"', '--compress=gzip', 'test'], + qr/\Qpg_dump: error: option --pipe is not supported with any compression type\E/, + 'pg_dump: option --pipe is not supported with any compression type' +); + +command_fails_like( + [ 'pg_dump', '-Fd', '--pipe="cat"', '-Z', '1', 'test'], + qr/\Qpg_dump: error: option --pipe is not supported with any compression type\E/, + 'pg_dump: option --pipe is not supported with any compression type' +); + +command_fails_like( + [ 'pg_dump', '-Fc', '--pipe="cat"', 'test'], + qr/\Qpg_dump: error: option --pipe is only supported with directory format\E/, + 'pg_dump: option --pipe is only supported with directory format' +); + +command_fails_like( + [ 'pg_dump', '--format=tar', '--pipe="cat"', 'test'], + qr/\Qpg_dump: error: option --pipe is only supported with directory format\E/, + 'pg_dump: option --pipe is only supported with directory format' +); + command_fails_like( [ 'pg_dump', '-j2', '--include-foreign-data=xxx' ], qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/, @@ -94,12 +136,38 @@ command_fails_like( command_fails_like( [ 'pg_restore', '-d', 'xxx', '-f', 'xxx' ], qr/\Qpg_restore: error: options -d\/--dbname and -f\/--file cannot be used together\E/, - 'pg_restore: options -d/--dbname and -f/--file cannot be used together'); + 'pg_restore: options -d/--dbname and -f/--file cannot be used together' +); + +command_fails_like( + [ 'pg_restore', '-f', '-', '--pipe="cat"', 'dumpdir' ], + qr/\Qpg_restore: error: cannot specify both an input file and --pipe\E/, + 'pg_restore: cannot specify both an input file and --pipe' +); + +command_fails_like( + [ 'pg_restore', '-Fd', '-f', '-', '--pipe="cat"', 'dumpdir' ], + qr/\Qpg_restore: error: cannot specify both an input file and --pipe\E/, + 'pg_restore: cannot specify both an input file and --pipe' +); + +command_fails_like( + [ 'pg_restore', '-Fc', '-f', '-', '--pipe="cat"' ], + qr/\Qpg_restore: error: option --pipe is only supported with directory format\E/, + 'pg_restore: option --pipe is only supported with directory format' +); + +command_fails_like( + [ 'pg_restore', '--format=tar', '-f', '-', '--pipe="cat"' ], + qr/\Qpg_restore: error: option --pipe is only supported with directory format\E/, + 'pg_restore: option --pipe is only supported with directory format' +); command_fails_like( [ 'pg_dump', '-c', '-a' ], qr/\Qpg_dump: error: options -c\/--clean and -a\/--data-only cannot be used together\E/, - 'pg_dump: options -c/--clean and -a/--data-only cannot be used together'); + 'pg_dump: options -c/--clean and -a/--data-only cannot be used together' +); command_fails_like( [ 'pg_dumpall', '-c', '-a' ], diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 3bc8e51561d..1388e6792b0 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -7,6 +7,7 @@ use warnings FATAL => 'all'; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +use File::Spec; my $tempdir = PostgreSQL::Test::Utils::tempdir; @@ -46,6 +47,24 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir; my $supports_icu = ($ENV{with_icu} eq 'yes'); my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1"); +# Use perl one-liner as a portable 'cat' replacement for Windows compatibility. +my $perlbin = $^X; +$perlbin =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; +my $perl_cat = "$perlbin -pe ''"; + +# Check for external gzip program for pipe tests. +my $gzip_bin = $ENV{GZIP_PROGRAM} || 'gzip'; +my $has_gzip_bin = (system("$gzip_bin --version >" . File::Spec->devnull() . " 2>&1") == 0); + +# Create output directories for pipe tests +mkdir "$tempdir/pipe_out_dir_parallel"; +mkdir "$tempdir/pipe_out_dir_parallel_8"; +mkdir "$tempdir/pipe_out_dir_complex"; +mkdir "$tempdir/pipe_out_dir_lo"; +mkdir "$tempdir/pipe_cross_dump"; +mkdir "$tempdir/pipe_cross_restore"; +mkdir "$tempdir/schema_only_pipe_dir"; + my %pgdump_runs = ( binary_upgrade => { dump_cmd => [ @@ -223,6 +242,140 @@ my %pgdump_runs = ( ], }, + defaults_dir_format_pipe => { + test_key => 'defaults', + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--pipe' => "$perl_cat > $tempdir/defaults_dir_format/%f", + '--statistics', + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_dir_format_pipe.sql", + '--pipe' => "$perl_cat $tempdir/defaults_dir_format/%f", + '--statistics', + ], + }, + + defaults_dir_format_pipe_dump_only => { + test_key => 'defaults', + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--pipe' => "$perl_cat > $tempdir/pipe_cross_dump/%f", + '--statistics', + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_dir_format_pipe_dump_only.sql", + '--statistics', + "$tempdir/pipe_cross_dump", + ], + }, + + defaults_dir_format_pipe_restore_only => { + test_key => 'defaults', + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--file' => "$tempdir/pipe_cross_restore", + '--statistics', + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_dir_format_pipe_restore_only.sql", + '--pipe' => ($supports_gzip && !$PostgreSQL::Test::Utils::windows_os) + ? "if [ -f $tempdir/pipe_cross_restore/%f.gz ]; then $gzip_bin -d -c $tempdir/pipe_cross_restore/%f.gz; else $perl_cat $tempdir/pipe_cross_restore/%f; fi" + : "$perl_cat $tempdir/pipe_cross_restore/%f", + '--statistics', + ], + }, + + defaults_parallel_pipe => { + test_key => 'defaults', + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--jobs' => 2, + '--pipe' => "$perl_cat > $tempdir/pipe_out_dir_parallel/%f", + '--statistics', + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_parallel_pipe.sql", + '--pipe' => "$perl_cat $tempdir/pipe_out_dir_parallel/%f", + '--statistics', + ], + }, + + defaults_parallel_8_pipe => { + test_key => 'defaults', + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--jobs' => 8, + '--pipe' => "$perl_cat > $tempdir/pipe_out_dir_parallel_8/%f", + '--statistics', + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_parallel_8_pipe.sql", + '--pipe' => "$perl_cat $tempdir/pipe_out_dir_parallel_8/%f", + '--statistics', + ], + }, + + defaults_complex_pipe => { + test_key => 'defaults', + skip_unless => \$has_gzip_bin, + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--pipe' => "$gzip_bin | $perl_cat > $tempdir/pipe_out_dir_complex/%f.gz", + '--statistics', + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_complex_pipe.sql", + '--pipe' => "$perl_cat $tempdir/pipe_out_dir_complex/%f.gz | $gzip_bin -d", + '--statistics', + ], + }, + defaults_lo_pipe => { + test_key => 'defaults', + dump_cmd => [ + 'pg_dump', + '--format' => 'directory', + '--statistics', + '--pipe' => "$perl_cat > $tempdir/pipe_out_dir_lo/%f", + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/defaults_lo_pipe.sql", + '--statistics', + '--pipe' => "$perl_cat $tempdir/pipe_out_dir_lo/%f", + ], + glob_patterns => [ + "$tempdir/pipe_out_dir_lo/toc.dat", + "$tempdir/pipe_out_dir_lo/blobs_*.toc", + ], + }, + # Do not use --no-sync to give test coverage for data sync. defaults_parallel => { test_key => 'defaults', @@ -527,6 +680,22 @@ my %pgdump_runs = ( 'postgres', ], }, + schema_only_pipe => { + test_key => 'schema_only', + dump_cmd => [ + 'pg_dump', '--no-sync', + '--format' => 'directory', + '--schema-only', + '--pipe' => "$perl_cat > $tempdir/schema_only_pipe_dir/%f", + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + '--format' => 'directory', + '--file' => "$tempdir/schema_only_pipe.sql", + '--pipe' => "$perl_cat $tempdir/schema_only_pipe_dir/%f", + ], + }, section_pre_data => { dump_cmd => [ 'pg_dump', '--no-sync', @@ -5217,6 +5386,13 @@ foreach my $run (sort keys %pgdump_runs) my $test_key = $run; my $run_db = 'postgres'; + if (defined($pgdump_runs{$run}->{skip_unless}) && + !${ $pgdump_runs{$run}->{skip_unless} }) + { + note "skipping run $run"; + next; + } + $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} }, "$run: pg_dump runs"); @@ -5334,6 +5510,57 @@ foreach my $run (sort keys %pgdump_runs) } } +######################################### +# Test error reporting for a failing pipe command. +# We use a perl one-liner that exits with 1 after processing input. +# This ensures we test the error handling in pclose() at the end of the dump, +# verifying that the child's exit status is correctly captured and reported. +my $failing_perl_cat = "$perlbin -pe \"END { exit 1 }\""; + +$node->command_fails_like( + [ 'pg_dump', '-Fd', "--pipe=$failing_perl_cat > %f", 'postgres' ], + qr/pipe command failed/, + 'pg_dump pipe command error reporting' +); + +$node->command_fails_like( + [ 'pg_restore', '-Fd', '-l', "--pipe=$failing_perl_cat " . "$tempdir/pipe_cross_dump" . '/%f' ], + qr/pipe command failed/, + 'pg_restore pipe command error reporting' +); + +# Targeted Edge Case Tests +$node->command_fails_like( + [ 'pg_dump', '-Fd', '--pipe=/nonexistent/binary', 'postgres' ], + qr/could not write to file: Broken pipe|Permission denied/, + 'pg_dump early pipe command execution failure' +); + +$node->command_fails_like( + [ 'pg_dump', '-Fd', '--pipe=no_such_command_at_all', 'postgres' ], + qr/could not write to file: Broken pipe|not found/, + 'pg_dump command not found error reporting' +); + +$node->command_fails_like( + [ 'pg_dump', '-Fd', '-f', '-', "--pipe=$perl_cat > %f", 'postgres' ], + qr/options -f\/--file and --pipe cannot be used together/, + 'pg_dump options -f/--file and --pipe conflict check' +); + +# Test that pg_restore rejects a positional argument when --pipe is used. +# We create a dummy cluster archive (containing toc.glo) to verify that +# even in cluster mode, the mutual exclusivity holds. +mkdir "$tempdir/dummy_cluster_archive"; +open my $fh, '>', "$tempdir/dummy_cluster_archive/toc.glo"; +close $fh; + +$node->command_fails_like( + [ 'pg_restore', '-Fd', '-l', "--pipe=$perl_cat %f", "$tempdir/dummy_cluster_archive" ], + qr/cannot specify both an input file and --pipe/, + 'pg_restore --pipe rejects positional argument even for cluster archive' +); + ######################################### # Stop the database instance, which will be removed at the end of the tests. diff --git a/src/bin/pg_dump/t/004_pg_dump_parallel.pl b/src/bin/pg_dump/t/004_pg_dump_parallel.pl index 738f34b1c1b..efcff1272d9 100644 --- a/src/bin/pg_dump/t/004_pg_dump_parallel.pl +++ b/src/bin/pg_dump/t/004_pg_dump_parallel.pl @@ -8,9 +8,15 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +# Use perl one-liner as a portable 'cat' replacement for Windows compatibility. +my $perlbin = $^X; +$perlbin =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; +my $perl_cat = "$perlbin -pe ''"; + my $dbname1 = 'regression_src'; my $dbname2 = 'regression_dest1'; my $dbname3 = 'regression_dest2'; +my $dbname4 = 'regression_dest3'; my $node = PostgreSQL::Test::Cluster->new('main'); $node->init; @@ -21,6 +27,7 @@ my $backupdir = $node->backup_dir; $node->run_log([ 'createdb', $dbname1 ]); $node->run_log([ 'createdb', $dbname2 ]); $node->run_log([ 'createdb', $dbname3 ]); +$node->run_log([ 'createdb', $dbname4 ]); $node->safe_psql( $dbname1, @@ -87,4 +94,27 @@ $node->command_ok( ], 'parallel restore as inserts'); +mkdir "$backupdir/dump_pipe"; + +$node->command_ok( + [ + 'pg_dump', + '--format' => 'directory', + '--no-sync', + '--jobs' => 2, + '--pipe' => "$perl_cat > $backupdir/dump_pipe/%f", + $node->connstr($dbname1), + ], + 'parallel dump with pipe'); + +$node->command_ok( + [ + 'pg_restore', '--verbose', + '--dbname' => $node->connstr($dbname4), + '--format' => 'directory', + '--jobs' => 3, + '--pipe' => "$perl_cat $backupdir/dump_pipe/%f", + ], + 'parallel restore with pipe'); + done_testing(); diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl index b2630ef2897..83358696e18 100644 --- a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl +++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl @@ -8,6 +8,11 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +# Use perl one-liner as a portable 'cat' replacement for Windows compatibility. +my $perlbin = $^X; +$perlbin =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; +my $perl_cat = "$perlbin -pe ''"; + my $tempdir = PostgreSQL::Test::Utils::tempdir; my $inputfile; @@ -98,6 +103,19 @@ command_ok( ], "filter file without patterns"); +mkdir "$backupdir/dump_pipe_filter"; + +command_ok( + [ + 'pg_dump', + '--port' => $port, + '--format' => 'directory', + '--pipe' => "$perl_cat > $backupdir/dump_pipe_filter/%f", + '--filter' => "$tempdir/inputfile.txt", + 'postgres' + ], + "filter file without patterns with pipe"); + my $dump = slurp_file($plainfile); like($dump, qr/^CREATE TABLE public\.table_one/m, "table one dumped"); -- 2.54.0.545.g6539524ca2-goog