From be1c2d894839531df520a7552068349654b0e0cb Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 7 May 2020 18:10:30 -0400 Subject: [PATCH v2 08/11] Create and use bbarchiver implementations for tar and tar sizing. --- src/backend/replication/Makefile | 1 + src/backend/replication/basebackup.c | 428 ++++++++++------------- src/backend/replication/basebackup_tar.c | 266 ++++++++++++++ 3 files changed, 454 insertions(+), 241 deletions(-) create mode 100644 src/backend/replication/basebackup_tar.c diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile index aacccd350d..6b3c77f2c0 100644 --- a/src/backend/replication/Makefile +++ b/src/backend/replication/Makefile @@ -21,6 +21,7 @@ OBJS = \ basebackup_libpq.o \ basebackup_progress.o \ basebackup_sink.o \ + basebackup_tar.o \ basebackup_throttle.o \ repl_gram.o \ slot.o \ diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 51c523e4ae..e8183daa68 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -27,7 +27,7 @@ #include "port.h" #include "postmaster/syslogger.h" #include "replication/basebackup.h" -#include "replication/basebackup_sink.h" +#include "replication/basebackup_archiver.h" #include "replication/backup_manifest.h" #include "replication/walsender.h" #include "replication/walsender_private.h" @@ -56,20 +56,26 @@ typedef struct pg_checksum_type manifest_checksum_type; } basebackup_options; -static int64 sendTablespace(bbsink *sink, char *path, char *oid, bool sizeonly, - struct backup_manifest_info *manifest); -static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, - List *tablespaces, bool sendtblspclinks, - backup_manifest_info *manifest, const char *spcoid); -static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, - struct stat *statbuf, bool missing_ok, Oid dboid, - backup_manifest_info *manifest, const char *spcoid); -static void sendFileWithContent(bbsink *sink, const char *filename, - const char *content, - backup_manifest_info *manifest); -static int64 _tarWriteHeader(bbsink *sink, const char *filename, - const char *linktarget, struct stat *statbuf, - bool sizeonly); +static void archive_database_cluster(List *tablespaces, bbarchiver *archiver, + StringInfo labelfile, + StringInfo tblspc_map_file, + bool leave_main_tablespace_open, + backup_manifest_info *manifest); +static void archive_tablespace(bbarchiver *archiver, char *path, char *oid, + struct backup_manifest_info *manifest); +static void archive_directory(bbarchiver *archiver, const char *path, + int basepathlen, List *tablespaces, + bool sendtblspclinks, + backup_manifest_info *manifest, + const char *spcoid); +static void archive_file(bbarchiver *archiver, const char *readfilename, + const char *tarfilename, struct stat *statbuf, + bool missing_ok, Oid dboid, + backup_manifest_info *manifest, const char *spcoid); +static void archive_file_with_content(bbarchiver *archiver, + const char *filename, + const char *content, + backup_manifest_info *manifest); static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf); static void perform_base_backup(basebackup_options *opt); static void parse_basebackup_options(List *options, basebackup_options *opt); @@ -233,6 +239,8 @@ perform_base_backup(basebackup_options *opt) List *tablespaces = NIL; bbsink *sink = bbsink_libpq_new(); bbsink *progress_sink; + bbarchiver *archiver; + bbarchiver *size_archiver; /* Set up network throttling, if client requested it */ if (opt->maxrate > 0) @@ -241,6 +249,10 @@ perform_base_backup(basebackup_options *opt) /* Set up progress reporting. */ sink = progress_sink = bbsink_progress_new(sink, opt->progress); + /* Set up tar archiving. */ + archiver = bbarchiver_tar_new(sink); + size_archiver = bbarchiver_tarsize_new(); + /* we're going to use a BufFile, so we need a ResourceOwner */ Assert(CurrentResourceOwner == NULL); CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup"); @@ -260,6 +272,8 @@ perform_base_backup(basebackup_options *opt) startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli, labelfile, &tablespaces, tblspc_map_file, opt->sendtblspcmapfile); + if (!opt->sendtblspcmapfile) + tblspc_map_file = NULL; /* * Once do_pg_start_backup has been called, ensure that any failure causes @@ -270,7 +284,6 @@ perform_base_backup(basebackup_options *opt) PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false)); { - ListCell *lc; tablespaceinfo *ti; /* @@ -290,89 +303,26 @@ perform_base_backup(basebackup_options *opt) ti->size = -1; tablespaces = lappend(tablespaces, ti); - /* - * Calculate the total backup size by summing up the size of each - * tablespace - */ + /* estimate sizes of all tablespaces, if PROGRESS option was given */ if (opt->progress) { basebackup_progress_estimate_backup_size(); - foreach(lc, tablespaces) - { - tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc); - - if (tmp->path == NULL) - tmp->size = sendDir(sink, ".", 1, true, tablespaces, true, NULL, - NULL); - else - tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true, - NULL); - } + archive_database_cluster(tablespaces, size_archiver, labelfile, + tblspc_map_file, false, NULL); } /* notify basebackup sink about start of backup */ bbsink_begin_backup(sink, startptr, starttli, tablespaces); - /* Send off our tablespaces one by one */ - foreach(lc, tablespaces) - { - tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc); - - if (ti->path == NULL) - { - struct stat statbuf; - bool sendtblspclinks = true; - - bbsink_begin_archive(sink, "base.tar"); - - /* In the main tar, include the backup_label first... */ - sendFileWithContent(sink, BACKUP_LABEL_FILE, labelfile->data, - &manifest); - - /* Then the tablespace_map file, if required... */ - if (opt->sendtblspcmapfile) - { - sendFileWithContent(sink, TABLESPACE_MAP, tblspc_map_file->data, - &manifest); - sendtblspclinks = false; - } - - /* Then the bulk of the files... */ - sendDir(sink, ".", 1, false, tablespaces, sendtblspclinks, - &manifest, NULL); - - /* ... and pg_control after everything else. */ - if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not stat file \"%s\": %m", - XLOG_CONTROL_FILE))); - sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, - false, InvalidOid, &manifest, NULL); - } - else - { - char *archive_name = psprintf("%s.tar", ti->oid); - - bbsink_begin_archive(sink, archive_name); - - sendTablespace(sink, ti->path, ti->oid, false, &manifest); - } - - /* - * If we're including WAL, and this is the main data directory we - * don't treat this as the end of the tablespace. Instead, we will - * include the xlog files below and stop afterwards. This is safe - * since the main data directory is always sent *last*. - */ - if (opt->includewal && ti->path == NULL) - { - Assert(lnext(tablespaces, lc) == NULL); - } - else - bbsink_end_archive(sink); - } + /* + * Back up all of the tablespaces. + * + * If the backup is to include WAL, leave the main tablespace open, + * so that we can archive the WAL files as well. + */ + archive_database_cluster(tablespaces, archiver, labelfile, + tblspc_map_file, opt->includewal, &manifest); basebackup_progress_wait_wal_archive(progress_sink); endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli); @@ -540,15 +490,14 @@ perform_base_backup(basebackup_options *opt) } /* send the WAL file itself */ - _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false); - + bbarchiver_begin_file(archiver, pathbuf, &statbuf); while ((cnt = basebackup_read_file(fd, buf, Min(sizeof(buf), wal_segment_size - len), len, pathbuf, true)) > 0) { CheckXLogRemoved(segno, tli); - bbsink_archive_contents(sink, buf, cnt); + bbarchiver_file_contents(archiver, buf, cnt); len += cnt; @@ -564,11 +513,7 @@ perform_base_backup(basebackup_options *opt) errmsg("unexpected WAL file size \"%s\"", walFileName))); } - /* - * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need - * for padding. - */ - Assert(wal_segment_size % TAR_BLOCK_SIZE == 0); + bbarchiver_end_file(archiver); CloseTransientFile(fd); @@ -579,7 +524,7 @@ perform_base_backup(basebackup_options *opt) * complete segment. */ StatusFilePath(pathbuf, walFileName, ".done"); - sendFileWithContent(sink, pathbuf, "", &manifest); + archive_file_with_content(archiver, pathbuf, "", &manifest); } /* @@ -602,12 +547,12 @@ perform_base_backup(basebackup_options *opt) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", pathbuf))); - sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid, - &manifest, NULL); + archive_file(archiver, pathbuf, pathbuf, &statbuf, false, + InvalidOid, &manifest, NULL); /* unconditionally mark file as archived */ StatusFilePath(pathbuf, fname, ".done"); - sendFileWithContent(sink, pathbuf, "", &manifest); + archive_file_with_content(archiver, pathbuf, "", &manifest); } bbsink_end_archive(sink); @@ -636,6 +581,74 @@ perform_base_backup(basebackup_options *opt) basebackup_progress_done(); } +/* + * Iterate over the entire cluster and feed each tablespace to the archiver + * in turn. + */ +static void +archive_database_cluster(List *tablespaces, bbarchiver *archiver, + StringInfo labelfile, StringInfo tblspc_map_file, + bool leave_main_tablespace_open, + backup_manifest_info *manifest) +{ + ListCell *lc; + + /* Send off our tablespaces one by one */ + foreach(lc, tablespaces) + { + tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc); + + bbarchiver_begin_tablespace(archiver, ti); + + if (ti->path == NULL) + { + struct stat statbuf; + bool sendtblspclinks = true; + + /* For the main tablespace, archive the backup_label first... */ + archive_file_with_content(archiver, BACKUP_LABEL_FILE, + labelfile->data, manifest); + + /* Then the tablespace_map file, if present... */ + if (tblspc_map_file != NULL) + { + archive_file_with_content(archiver, TABLESPACE_MAP, + tblspc_map_file->data, + manifest); + sendtblspclinks = false; + } + + /* Then the bulk of the files... */ + archive_directory(archiver, ".", 1, tablespaces, + sendtblspclinks, manifest, NULL); + + /* ... and pg_control after everything else. */ + if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + XLOG_CONTROL_FILE))); + archive_file(archiver, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, + &statbuf, false, InvalidOid, manifest, NULL); + } + else + archive_tablespace(archiver, ti->path, ti->oid, manifest); + + /* + * If we were asked to leave the main tablespace open, then do so. + * This is safe since the main data directory is always sent *last*, + * so we'll not try to begin another tablespace without ending this + * one. + */ + if (leave_main_tablespace_open && ti->path == NULL) + { + Assert(lnext(tablespaces, lc) == NULL); + } + else + bbarchiver_end_tablespace(archiver); + } +} + /* * list_sort comparison function, to compare log/seg portion of WAL segment * filenames, ignoring the timeline portion. @@ -846,18 +859,20 @@ SendBaseBackup(BaseBackupCmd *cmd) } /* - * Inject a file with given name and content in the output tar stream. + * Feed a file to the archiver that does not actually exist in the source + * directory. We use this to inject things like the backup_label file into + * the backup. */ static void -sendFileWithContent(bbsink *sink, const char *filename, const char *content, - backup_manifest_info *manifest) +archive_file_with_content(bbarchiver *archiver, const char *filename, + const char *content, backup_manifest_info *manifest) { struct stat statbuf; - int pad, - len; + int len; pg_checksum_context checksum_ctx; - pg_checksum_init(&checksum_ctx, manifest->checksum_type); + if (manifest != NULL) + pg_checksum_init(&checksum_ctx, manifest->checksum_type); len = strlen(content); @@ -877,22 +892,17 @@ sendFileWithContent(bbsink *sink, const char *filename, const char *content, statbuf.st_mode = pg_file_create_mode; statbuf.st_size = len; - _tarWriteHeader(sink, filename, NULL, &statbuf, false); - bbsink_archive_contents(sink, content, len); + bbarchiver_begin_file(archiver, filename, &statbuf); + if (bbarchiver_needs_file_contents(archiver)) + bbarchiver_file_contents(archiver, content, len); + bbarchiver_end_file(archiver); - /* Pad to a multiple of the tar block size. */ - pad = tarPaddingBytesRequired(len); - if (pad > 0) + if (manifest != NULL) { - char buf[TAR_BLOCK_SIZE]; - - MemSet(buf, 0, pad); - bbsink_archive_contents(sink, buf, pad); + pg_checksum_update(&checksum_ctx, (uint8 *) content, len); + AddFileToBackupManifest(manifest, NULL, filename, len, + (pg_time_t) statbuf.st_mtime, &checksum_ctx); } - - pg_checksum_update(&checksum_ctx, (uint8 *) content, len); - AddFileToBackupManifest(manifest, NULL, filename, len, - (pg_time_t) statbuf.st_mtime, &checksum_ctx); } /* @@ -902,11 +912,10 @@ sendFileWithContent(bbsink *sink, const char *filename, const char *content, * * Only used to send auxiliary tablespaces, not PGDATA. */ -static int64 -sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly, - backup_manifest_info *manifest) +static void +archive_tablespace(bbarchiver *archiver, char *path, char *spcoid, + backup_manifest_info *manifest) { - int64 size; char pathbuf[MAXPGPATH]; struct stat statbuf; @@ -930,17 +939,14 @@ sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly, pathbuf))); /* If the tablespace went away while scanning, it's no error. */ - return 0; + return; } - size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf, - sizeonly); + bbarchiver_directory(archiver, TABLESPACE_VERSION_DIRECTORY, &statbuf); /* Send all the files in the tablespace version directory */ - size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest, - spcoid); - - return size; + archive_directory(archiver, pathbuf, strlen(path), NIL, true, manifest, + spcoid); } /* @@ -955,16 +961,15 @@ sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly, * information in the tar file. If not, we can skip that * as it will be sent separately in the tablespace_map file. */ -static int64 -sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, - List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, - const char *spcoid) +static void +archive_directory(bbarchiver *archiver, const char *path, int basepathlen, + List *tablespaces, bool sendtblspclinks, + backup_manifest_info *manifest, const char *spcoid) { DIR *dir; struct dirent *de; char pathbuf[MAXPGPATH * 2]; struct stat statbuf; - int64 size = 0; const char *lastDir; /* Split last dir from parent path. */ bool isDbDir = false; /* Does this directory contain relations? */ @@ -1117,8 +1122,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, { elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name); convert_link_to_directory(pathbuf, &statbuf); - size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, - &statbuf, sizeonly); + bbarchiver_directory(archiver, pathbuf + basepathlen + 1, + &statbuf); excludeFound = true; break; } @@ -1135,8 +1140,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, { elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath); convert_link_to_directory(pathbuf, &statbuf); - size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, - &statbuf, sizeonly); + bbarchiver_directory(archiver, pathbuf + basepathlen + 1, + &statbuf); continue; } @@ -1149,15 +1154,15 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, { /* If pg_wal is a symlink, write it as a directory anyway */ convert_link_to_directory(pathbuf, &statbuf); - size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, - &statbuf, sizeonly); + bbarchiver_directory(archiver, pathbuf + basepathlen + 1, + &statbuf); /* * Also send archive_status directory (by hackishly reusing * statbuf from above ...). */ - size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL, - &statbuf, sizeonly); + bbarchiver_directory(archiver, "./pg_wal/archive_status", + &statbuf); continue; /* don't recurse into pg_wal */ } @@ -1188,8 +1193,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, pathbuf))); linkpath[rllen] = '\0'; - size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath, - &statbuf, sizeonly); + bbarchiver_symbolic_link(archiver, pathbuf + basepathlen + 1, + linkpath, &statbuf); #else /* @@ -1212,8 +1217,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, * Store a directory entry in the tar file so we can get the * permissions right. */ - size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf, - sizeonly); + bbarchiver_directory(archiver, pathbuf + basepathlen + 1, + &statbuf); /* * Call ourselves recursively for a directory, unless it happens @@ -1244,36 +1249,21 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, skip_this_dir = true; if (!skip_this_dir) - size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces, - sendtblspclinks, manifest, spcoid); + archive_directory(archiver, pathbuf, basepathlen, tablespaces, + sendtblspclinks, manifest, spcoid); } else if (S_ISREG(statbuf.st_mode)) { - bool sent = false; - - if (!sizeonly) - sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf, - true, isDbDir ? atooid(lastDir + 1) : InvalidOid, - manifest, spcoid); - - if (sent || sizeonly) - { - /* Add size. */ - size += statbuf.st_size; - - /* Pad to a multiple of the tar block size. */ - size += tarPaddingBytesRequired(statbuf.st_size); - - /* Size of the header for the file. */ - size += TAR_BLOCK_SIZE; - } + archive_file(archiver, pathbuf, pathbuf + basepathlen + 1, + &statbuf, true, + isDbDir ? atooid(lastDir + 1) : InvalidOid, + manifest, spcoid); } else ereport(WARNING, (errmsg("skipping special file \"%s\"", pathbuf))); } FreeDir(dir); - return size; } /* @@ -1324,14 +1314,11 @@ is_checksummed_file(const char *fullpath, const char *filename) * * If dboid is anything other than InvalidOid then any checksum failures detected * will get reported to the stats collector. - * - * Returns true if the file was successfully sent, false if 'missing_ok', - * and the file did not exist. */ -static bool -sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, - struct stat *statbuf, bool missing_ok, Oid dboid, - backup_manifest_info *manifest, const char *spcoid) +static void +archive_file(bbarchiver *archiver, const char *readfilename, + const char *tarfilename, struct stat *statbuf, bool missing_ok, + Oid dboid, backup_manifest_info *manifest, const char *spcoid) { int fd; BlockNumber blkno = 0; @@ -1343,27 +1330,33 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, int i; pgoff_t len = 0; char *page; - size_t pad; PageHeader phdr; int segmentno = 0; char *segmentpath; bool verify_checksum = false; pg_checksum_context checksum_ctx; - pg_checksum_init(&checksum_ctx, manifest->checksum_type); + if (manifest != NULL) + pg_checksum_init(&checksum_ctx, manifest->checksum_type); + + bbarchiver_begin_file(archiver, tarfilename, statbuf); + + if (!bbarchiver_needs_file_contents(archiver)) + { + bbarchiver_end_file(archiver); + return; + } fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY); if (fd < 0) { if (errno == ENOENT && missing_ok) - return false; + return; ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", readfilename))); } - _tarWriteHeader(sink, tarfilename, NULL, statbuf, false); - if (!noverify_checksums && DataChecksumsEnabled()) { char *filename; @@ -1517,10 +1510,11 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, } } - bbsink_archive_contents(sink, buf, cnt); + bbarchiver_file_contents(archiver, buf, cnt); /* Also feed it to the checksum machinery. */ - pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt); + if (manifest != NULL) + pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt); len += cnt; } @@ -1532,23 +1526,14 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, while (len < statbuf->st_size) { cnt = Min(sizeof(buf), statbuf->st_size - len); - bbsink_archive_contents(sink, buf, cnt); - pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt); + bbarchiver_file_contents(archiver, buf, cnt); + if (manifest != NULL) + pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt); len += cnt; } } - /* - * Pad to a block boundary, per tar format requirements. (This small - * piece of data is probably not worth throttling, and is not checksummed - * because it's not actually part of the file.) - */ - pad = tarPaddingBytesRequired(len); - if (pad > 0) - { - MemSet(buf, 0, pad); - bbsink_archive_contents(sink, buf, pad); - } + bbarchiver_end_file(archiver); CloseTransientFile(fd); @@ -1565,54 +1550,15 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, total_checksum_failures += checksum_failures; - AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size, - (pg_time_t) statbuf->st_mtime, &checksum_ctx); - - return true; -} - - -static int64 -_tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, - struct stat *statbuf, bool sizeonly) -{ - char h[TAR_BLOCK_SIZE]; - enum tarError rc; - - if (!sizeonly) - { - rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_mtime); - - switch (rc) - { - case TAR_OK: - break; - case TAR_NAME_TOO_LONG: - ereport(ERROR, - (errmsg("file name too long for tar format: \"%s\"", - filename))); - break; - case TAR_SYMLINK_TOO_LONG: - ereport(ERROR, - (errmsg("symbolic link target too long for tar format: " - "file name \"%s\", target \"%s\"", - filename, linktarget))); - break; - default: - elog(ERROR, "unrecognized tar error: %d", rc); - } - - bbsink_archive_contents(sink, h, sizeof(h)); - } - - return sizeof(h); + if (manifest != NULL) + AddFileToBackupManifest(manifest, spcoid, tarfilename, + statbuf->st_size, + (pg_time_t) statbuf->st_mtime, &checksum_ctx); } /* * If the entry in statbuf is a link, then adjust statbuf to make it look like a - * directory, so that it will be written that way. + * directory. */ static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf) diff --git a/src/backend/replication/basebackup_tar.c b/src/backend/replication/basebackup_tar.c new file mode 100644 index 0000000000..8618f2a0ec --- /dev/null +++ b/src/backend/replication/basebackup_tar.c @@ -0,0 +1,266 @@ +#include "postgres.h" + +#include "pgtar.h" +#include "replication/basebackup_archiver.h" + +typedef struct bbarchiver_tar +{ + bbarchiver base; + bbsink *sink; + size_t file_len; +} bbarchiver_tar; + +typedef struct bbarchiver_tarsize +{ + bbarchiver base; + tablespaceinfo *tsinfo; +} bbarchiver_tarsize; + +static void bbarchiver_tar_begin_tablespace(bbarchiver *archiver, + tablespaceinfo *tsinfo); +static void bbarchiver_tar_end_tablespace(bbarchiver *archiver); +static void bbarchiver_tar_begin_file(bbarchiver *archiver, + const char *relative_path, + struct stat *statbuf); +static void bbarchiver_tar_file_contents(bbarchiver *archiver, + const char *data, + size_t len); +static void bbarchiver_tar_end_file(bbarchiver *archiver); +static void bbarchiver_tar_directory(bbarchiver *archiver, + const char *relative_path, + struct stat *statbuf); +static void bbarchiver_tar_symbolic_link(bbarchiver *archiver, + const char *relative_path, + const char *linktarget, + struct stat *statbuf); +static void report_tar_error(enum tarError rc, const char *filename, + const char *linktarget); + +static void bbarchiver_tarsize_begin_tablespace(bbarchiver *archiver, + tablespaceinfo *tsinfo); +static void bbarchiver_tarsize_begin_file(bbarchiver *archiver, + const char *relative_path, + struct stat *statbuf); +static void bbarchiver_tarsize_directory(bbarchiver *archiver, + const char *relative_path, + struct stat *statbuf); +static void bbarchiver_tarsize_symbolic_link(bbarchiver *archiver, + const char *relative_path, + const char *linktarget, + struct stat *statbuf); +static void add_tar_size(bbarchiver *archiver, uint64 file_size); + +const bbarchiver_ops bbarchiver_tar_ops = { + .begin_tablespace = bbarchiver_tar_begin_tablespace, + .end_tablespace = bbarchiver_tar_end_tablespace, + .begin_file = bbarchiver_tar_begin_file, + .file_contents = bbarchiver_tar_file_contents, + .end_file = bbarchiver_tar_end_file, + .directory = bbarchiver_tar_directory, + .symbolic_link = bbarchiver_tar_symbolic_link, +}; + +const bbarchiver_ops bbarchiver_tarsize_ops = { + .begin_tablespace = bbarchiver_tarsize_begin_tablespace, + .end_tablespace = bbarchiver_noop_end_tablespace, + .begin_file = bbarchiver_tarsize_begin_file, + .file_contents = NULL, + .end_file = bbarchiver_noop_end_file, + .directory = bbarchiver_tarsize_directory, + .symbolic_link = bbarchiver_tarsize_symbolic_link, +}; + +bbarchiver * +bbarchiver_tar_new(bbsink *sink) +{ + bbarchiver_tar *archiver = palloc0(sizeof(bbarchiver_tar)); + + *((const bbarchiver_ops **) &archiver->base.bba_ops) = &bbarchiver_tar_ops; + archiver->base.bba_next = NULL; + archiver->sink = sink; + + return &archiver->base; +} + +static void +bbarchiver_tar_begin_tablespace(bbarchiver *archiver, tablespaceinfo *tsinfo) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + char *archive_name = "base.tar"; + + if (tsinfo->path != NULL) + archive_name = psprintf("%s.tar", tsinfo->oid); + + bbsink_begin_archive(myarchiver->sink, archive_name); +} + +static void +bbarchiver_tar_end_tablespace(bbarchiver *archiver) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + + bbsink_end_archive(myarchiver->sink); +} + +static void +bbarchiver_tar_begin_file(bbarchiver *archiver, const char *relative_path, + struct stat *statbuf) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + char h[TAR_BLOCK_SIZE]; + enum tarError rc; + + myarchiver->file_len = 0; + + rc = tarCreateHeader(h, relative_path, NULL, statbuf->st_size, + statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, + statbuf->st_mtime); + if (rc != TAR_OK) + report_tar_error(rc, relative_path, NULL); + + bbsink_archive_contents(myarchiver->sink, h, sizeof(h)); +} + +static void +bbarchiver_tar_file_contents(bbarchiver *archiver, const char *data, + size_t len) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + + myarchiver->file_len += len; + bbsink_archive_contents(myarchiver->sink, data, len); +} + +static void +bbarchiver_tar_end_file(bbarchiver *archiver) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + int pad; + + /* Pad to a block boundary, per tar format requirements. */ + pad = tarPaddingBytesRequired(myarchiver->file_len); + if (pad > 0) + { + char buf[TAR_BLOCK_SIZE]; + + MemSet(buf, 0, pad); + bbsink_archive_contents(myarchiver->sink, buf, pad); + } +} + +static void +bbarchiver_tar_directory(bbarchiver *archiver, const char *relative_path, + struct stat *statbuf) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + char h[TAR_BLOCK_SIZE]; + enum tarError rc; + + rc = tarCreateHeader(h, relative_path, NULL, statbuf->st_size, + statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, + statbuf->st_mtime); + if (rc != TAR_OK) + report_tar_error(rc, relative_path, NULL); + + bbsink_archive_contents(myarchiver->sink, h, sizeof(h)); +} + +static void +bbarchiver_tar_symbolic_link(bbarchiver *archiver, const char *relative_path, + const char *linktarget, struct stat *statbuf) +{ + bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver; + char h[TAR_BLOCK_SIZE]; + enum tarError rc; + + rc = tarCreateHeader(h, relative_path, linktarget, statbuf->st_size, + statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, + statbuf->st_mtime); + if (rc != TAR_OK) + report_tar_error(rc, relative_path, NULL); + + bbsink_archive_contents(myarchiver->sink, h, sizeof(h)); +} + +static void +report_tar_error(enum tarError rc, const char *filename, + const char *linktarget) +{ + switch (rc) + { + case TAR_OK: + break; + case TAR_NAME_TOO_LONG: + ereport(ERROR, + (errmsg("file name too long for tar format: \"%s\"", + filename))); + break; + case TAR_SYMLINK_TOO_LONG: + Assert(linktarget != NULL); + ereport(ERROR, + (errmsg("symbolic link target too long for tar format: " + "file name \"%s\", target \"%s\"", + filename, linktarget))); + break; + default: + elog(ERROR, "unrecognized tar error: %d", rc); + } +} + +/* + * Create an archiver that calculates an estimated size for a tar file built + * from the files visited. + */ +bbarchiver * +bbarchiver_tarsize_new(void) +{ + bbarchiver_tarsize *archiver = palloc0(sizeof(bbarchiver_tarsize)); + + *((const bbarchiver_ops **) &archiver->base.bba_ops) = + &bbarchiver_tarsize_ops; + archiver->base.bba_next = NULL; + + return &archiver->base; +} + +static void +bbarchiver_tarsize_begin_tablespace(bbarchiver *archiver, + tablespaceinfo *tsinfo) +{ + bbarchiver_tarsize *myarchiver = (bbarchiver_tarsize *) archiver; + + myarchiver->tsinfo = tsinfo; + tsinfo->size = 0; +} + +static void +bbarchiver_tarsize_begin_file(bbarchiver *archiver, const char *relative_path, + struct stat *statbuf) +{ + add_tar_size(archiver, statbuf->st_size); +} + +static void +bbarchiver_tarsize_directory(bbarchiver *archiver, const char *relative_path, + struct stat *statbuf) +{ + add_tar_size(archiver, 0); +} + +static void +bbarchiver_tarsize_symbolic_link(bbarchiver *archiver, + const char *relative_path, + const char *linktarget, + struct stat *statbuf) +{ + add_tar_size(archiver, 0); +} + +static void +add_tar_size(bbarchiver *archiver, uint64 file_size) +{ + bbarchiver_tarsize *myarchiver = (bbarchiver_tarsize *) archiver; + + myarchiver->tsinfo->size += + TAR_BLOCK_SIZE + file_size + tarPaddingBytesRequired(file_size); +} -- 2.24.3 (Apple Git-128)