From 9da0442e9f3f440d9e7a09f2ba2a4df7c10946ac Mon Sep 17 00:00:00 2001 From: Amul Sul Date: Tue, 2 Jul 2024 10:26:35 +0530 Subject: [PATCH v1 08/10] pg_verifybackup: Add backup format and compression option ---- NOTE: This patch is not meant to be committed separately. It should be squashed with the next patch that implements tar format support. ---- --- src/bin/pg_verifybackup/pg_verifybackup.c | 143 +++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c index 375f196b300..0d458298f34 100644 --- a/src/bin/pg_verifybackup/pg_verifybackup.c +++ b/src/bin/pg_verifybackup/pg_verifybackup.c @@ -18,6 +18,7 @@ #include #include +#include "common/compression.h" #include "common/parse_manifest.h" #include "fe_utils/simple_list.h" #include "getopt_long.h" @@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2, 3) pg_attribute_noreturn(); +static char find_backup_format(verifier_context *context); +static pg_compress_algorithm find_backup_compression(verifier_context *context); static void verify_backup_directory(verifier_context *context, char *relpath, char *fullpath); static void verify_backup_file(verifier_context *context, @@ -82,6 +85,9 @@ bool skip_checksums = false; static uint64 total_size = 0; static uint64 done_size = 0; +char format = '\0'; /* p(lain)/t(ar) */ +pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE; + /* * Main entry point. */ @@ -92,11 +98,13 @@ main(int argc, char **argv) {"exit-on-error", no_argument, NULL, 'e'}, {"ignore", required_argument, NULL, 'i'}, {"manifest-path", required_argument, NULL, 'm'}, + {"format", required_argument, NULL, 'F'}, {"no-parse-wal", no_argument, NULL, 'n'}, {"progress", no_argument, NULL, 'P'}, {"quiet", no_argument, NULL, 'q'}, {"skip-checksums", no_argument, NULL, 's'}, {"wal-directory", required_argument, NULL, 'w'}, + {"compress", required_argument, NULL, 'Z'}, {NULL, 0, NULL, 0} }; @@ -107,6 +115,7 @@ main(int argc, char **argv) bool quiet = false; char *wal_directory = NULL; char *pg_waldump_path = NULL; + bool tar_compression_specified = false; pg_logging_init(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup")); @@ -149,7 +158,7 @@ main(int argc, char **argv) simple_string_list_append(&context.ignore_list, "recovery.signal"); simple_string_list_append(&context.ignore_list, "standby.signal"); - while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1) + while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1) { switch (c) { @@ -168,6 +177,15 @@ main(int argc, char **argv) manifest_path = pstrdup(optarg); canonicalize_path(manifest_path); break; + case 'F': + if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0) + format = 'p'; + else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0) + format = 't'; + else + pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"", + optarg); + break; case 'n': no_parse_wal = true; break; @@ -184,6 +202,12 @@ main(int argc, char **argv) wal_directory = pstrdup(optarg); canonicalize_path(wal_directory); break; + case 'Z': + if (!parse_compress_algorithm(optarg, &compress_algorithm)) + pg_fatal("unrecognized compression algorithm: \"%s\"", + optarg); + tar_compression_specified = true; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -215,11 +239,41 @@ main(int argc, char **argv) pg_fatal("cannot specify both %s and %s", "-P/--progress", "-q/--quiet"); + /* Complain if compression method specified but the format isn't tar. */ + if (format != 't' && tar_compression_specified) + { + pg_log_error("only tar mode backups can be compressed"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + /* Determine the backup format if it hasn't been specified. */ + if (format == '\0') + format = find_backup_format(&context); + + /* + * Determine the tar backup compression method if it hasn't been + * specified. + */ + if (format == 't' && !tar_compression_specified) + compress_algorithm = find_backup_compression(&context); + /* Unless --no-parse-wal was specified, we will need pg_waldump. */ if (!no_parse_wal) { int ret; + /* + * XXX: In the future, we should consider enhancing pg_waldump to read + * WAL files from the tar archive. + */ + if (format != 'p') + { + pg_log_error("pg_waldump does not support parsing WAL files from a tar archive."); + pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname); + exit(1); + } + pg_waldump_path = pg_malloc(MAXPGPATH); ret = find_other_exec(argv[0], "pg_waldump", "pg_waldump (PostgreSQL) " PG_VERSION "\n", @@ -274,8 +328,13 @@ main(int argc, char **argv) /* * Now do the expensive work of verifying file checksums, unless we were * told to skip it. + * + * We were only checking the plain backup here. For the tar backup, file + * checksums verification (if requested) will be done immediately when the + * file is accessed, as we don't have random access to the files like we + * do with plain backups. */ - if (!skip_checksums) + if (!skip_checksums && format == 'p') verify_backup_checksums(&context); /* @@ -1041,6 +1100,84 @@ progress_report(bool finished) fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr); } +/* + * To detect the backup format, it checks for the PG_VERSION file in the backup + * directory. If found, it will be considered a plain-format backup; otherwise, + * it will be assumed to be a tar-format backup. + */ +static char +find_backup_format(verifier_context *context) +{ + char result; + char *path; + struct stat sb; + + /* Should be here only if the backup format is unknown */ + Assert(format == '\0'); + + /* Check PG_VERSION file. */ + path = psprintf("%s/%s", context->backup_directory, "PG_VERSION"); + result = (stat(path, &sb) == 0) ? 'p' : 't'; + pfree(path); + + return result; +} + +/* + * To determine the compression format, we will search for the main data + * directory archive and its extension, which starts with base.tar, as + * pg_basebackup writes the main data directory to an archive file named + * base.tar followed by a compression type extension like .gz, .lz4, or .zst. + */ +static pg_compress_algorithm +find_backup_compression(verifier_context *context) +{ + char *path; + struct stat sb; + bool found; + + /* Should be here only for tar backup */ + Assert(format == 't'); + + /* + * Is this a tar archive? + */ + path = psprintf("%s/%s", context->backup_directory, "base.tar"); + found = (stat(path, &sb) == 0); + pfree(path); + if (found) + return PG_COMPRESSION_NONE; + + /* + * Is this a .tar.gz archive? + */ + path = psprintf("%s/%s", context->backup_directory, "base.tar.gz"); + found = (stat(path, &sb) == 0); + pfree(path); + if (found) + return PG_COMPRESSION_GZIP; + + /* + * Is this a .tar.lz4 archive? + */ + path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4"); + found = (stat(path, &sb) == 0); + pfree(path); + if (found) + return PG_COMPRESSION_LZ4; + + /* + * Is this a .tar.zst archive? + */ + path = psprintf("%s/%s", context->backup_directory, "base.tar.zst"); + found = (stat(path, &sb) == 0); + pfree(path); + if (found) + return PG_COMPRESSION_ZSTD; + + return PG_COMPRESSION_NONE; /* placate compiler */ +} + /* * Print out usage information and exit. */ @@ -1053,11 +1190,13 @@ usage(void) printf(_(" -e, --exit-on-error exit immediately on error\n")); printf(_(" -i, --ignore=RELATIVE_PATH ignore indicated path\n")); printf(_(" -m, --manifest-path=PATH use specified path for manifest\n")); + printf(_(" -F, --format=p|t backup format (plain, tar)\n")); printf(_(" -n, --no-parse-wal do not try to parse WAL files\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -q, --quiet do not print any output, except for errors\n")); printf(_(" -s, --skip-checksums skip checksum verification\n")); printf(_(" -w, --wal-directory=PATH use specified path for WAL files\n")); + printf(_(" -Z, --compress=METHOD compress method (gzip, lz4, zstd, none) \n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); -- 2.18.0