From 370cc091048a2259e633ca65473ed1a835e13451 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 24 Jun 2015 15:10:16 +0900 Subject: [PATCH 8/8] Fix symlink usage in pg_rewind Make use of pg_readlink to determine where a symlink in a source PGDATA streamed is from. Also remove restrictions related to symlink comparisons on source and target instances, those need to be only limited to tablespaces. --- src/bin/pg_rewind/file_ops.c | 2 ++ src/bin/pg_rewind/filemap.c | 11 ++----- src/bin/pg_rewind/filemap.h | 1 + src/bin/pg_rewind/libpq_fetch.c | 63 +++++++++++++++++++++++++---------------- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index c2d8aa1..29ddb1d 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -139,6 +139,7 @@ remove_target(file_entry_t *entry) remove_target_file(entry->path); break; + case FILE_TYPE_TABLESPACE: case FILE_TYPE_SYMLINK: remove_target_symlink(entry->path); break; @@ -156,6 +157,7 @@ create_target(file_entry_t *entry) create_target_dir(entry->path); break; + case FILE_TYPE_TABLESPACE: case FILE_TYPE_SYMLINK: create_target_symlink(entry->path, entry->link_target); break; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index 05eff68..6ef2342 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -112,12 +112,6 @@ process_source_file(const char *path, file_type_t type, size_t newsize, switch (type) { case FILE_TYPE_DIRECTORY: - if (exists && !S_ISDIR(statbuf.st_mode)) - { - /* it's a directory in source, but not in target. Strange.. */ - pg_fatal("\"%s\" is not a directory\n", localpath); - } - if (!exists) action = FILE_ACTION_CREATE; else @@ -125,7 +119,7 @@ process_source_file(const char *path, file_type_t type, size_t newsize, oldsize = 0; break; - case FILE_TYPE_SYMLINK: + case FILE_TYPE_TABLESPACE: if (exists && #ifndef WIN32 !S_ISLNK(statbuf.st_mode) @@ -140,7 +134,8 @@ process_source_file(const char *path, file_type_t type, size_t newsize, */ pg_fatal("\"%s\" is not a symbolic link\n", localpath); } - + /* fallback to default */ + case FILE_TYPE_SYMLINK: if (!exists) action = FILE_ACTION_CREATE; else diff --git a/src/bin/pg_rewind/filemap.h b/src/bin/pg_rewind/filemap.h index 9943ec5..35540d6 100644 --- a/src/bin/pg_rewind/filemap.h +++ b/src/bin/pg_rewind/filemap.h @@ -36,6 +36,7 @@ typedef enum { FILE_TYPE_REGULAR, FILE_TYPE_DIRECTORY, + FILE_TYPE_TABLESPACE, FILE_TYPE_SYMLINK } file_type_t; diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index df71069..49950b9 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -140,30 +140,28 @@ libpqProcessFileList(void) * Create a recursive directory listing of the whole data directory. * * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. - * - * XXX: There is no backend function to get a symbolic link's target in - * general, so if the admin has put any custom symbolic links in the data - * directory, they won't be copied correctly. + * targets of the symlinks in the data directory. */ sql = "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.') AS filename) AS fn,\n" - " pg_stat_file(fn.filename) AS this\n" + " SELECT '' AS path, filename, size, isdir, islink FROM\n" + " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" + " pg_stat_file(fn.filename, true) AS this\n" + " WHERE this.size IS NOT NULL\n" " UNION ALL\n" " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" + " fn, this.size, this.isdir, this.islink\n" " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" - " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" - " AND oid::text = files.filename\n"; + " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" + " pg_stat_file(parent.path || parent.filename || '/' || fn, true)\n" + " AS this\n" + " WHERE parent.isdir = 't' AND\n" + " this.size IS NOT NULL)\n" + "SELECT path || filename AS file_path, size, isdir, islink,\n" + " path = 'pg_tblspc/' AS istblspc,\n" + " CASE WHEN islink THEN pg_readlink(path || filename)\n" + " ELSE NULL END AS link_target\n" + "FROM files\n"; res = PQexec(conn, sql); if (PQresultStatus(res) != PGRES_TUPLES_OK) @@ -171,7 +169,7 @@ libpqProcessFileList(void) PQresultErrorMessage(res)); /* sanity check the result set */ - if (PQnfields(res) != 4) + if (PQnfields(res) != 6) pg_fatal("unexpected result set while fetching file list\n"); /* Read result to local variables */ @@ -180,10 +178,14 @@ libpqProcessFileList(void) char *path = PQgetvalue(res, i, 0); int filesize = atoi(PQgetvalue(res, i, 1)); bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0); - char *link_target = PQgetvalue(res, i, 3); + bool islink = (strcmp(PQgetvalue(res, i, 3), "t") == 0); + bool istblspc = (strcmp(PQgetvalue(res, i, 4), "t") == 0); + char *link_target = PQgetvalue(res, i, 5); file_type_t type; - if (link_target[0]) + if (istblspc) + type = FILE_TYPE_TABLESPACE; + else if (islink) type = FILE_TYPE_SYMLINK; else if (isdir) type = FILE_TYPE_DIRECTORY; @@ -259,8 +261,7 @@ receiveFileChunks(const char *sql) } if (PQgetisnull(res, 0, 0) || - PQgetisnull(res, 0, 1) || - PQgetisnull(res, 0, 2)) + PQgetisnull(res, 0, 1)) { pg_fatal("unexpected null values in result while fetching remote files\n"); } @@ -278,6 +279,20 @@ receiveFileChunks(const char *sql) memcpy(filename, PQgetvalue(res, 0, 0), filenamelen); filename[filenamelen] = '\0'; + /* + * It may be possible that a file has been deleted on remote side after + * creating the file map. In this case simply ignore it and move on. + */ + if (PQgetisnull(res, 0, 2)) + { + pg_log(PG_DEBUG, + "received NULL chunk for file \"%s\", file has been deleted\n", + filename); + pg_free(filename); + PQclear(res); + continue; + } + chunk = PQgetvalue(res, 0, 2); pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %d, size %d\n", @@ -445,7 +460,7 @@ libpq_executeFileMap(filemap_t *map) */ sql = "SELECT path, begin, \n" - " pg_read_binary_file(path, begin, len) AS chunk\n" + " pg_read_binary_file(path, begin, len, true) AS chunk\n" "FROM fetchchunks\n"; receiveFileChunks(sql); -- 2.4.4