From f12a2cec0489d6c20ed55665bc6698ac701e5d73 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 23 Jun 2015 13:16:23 +0900 Subject: [PATCH 6/6] Make pg_rewind able to detect deleted files on remote source When getting a list of PGDATA files from a remote server with pg_rewind, it is possible that this list contains temporary files or files that have been deleted on the remote server after taking the file list, like what DROP TABLE would do for a relation. This can lead to errors when calling pg_read_binary_file on an entry that is listed but does not exist anymore, preventing the rewind from working correctly. In order to prevent such problems, use the new if_not_exists options in pg_ls_dir, pg_tablespace_location and pg_read_binary_file to not fail if a file does not exist anymore and continue process as for example a relation file removed does not prevent a rewound node from replaying WAL in recovery, node rewound guaranteed to have a minimum recovery point set to the current WAL location of the remote node that is put in sync with. --- src/bin/pg_rewind/libpq_fetch.c | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index 6ffd24e..2051d30 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -149,18 +149,24 @@ libpqProcessFileList(void) sql = "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.') AS filename) AS fn,\n" - " pg_stat_file(fn.filename) AS this\n" + " (SELECT filename\n" + " FROM pg_ls_dir('.', true, true) AS filename\n" + " WHERE filename NOT IN ('.', '..')) AS fn,\n" + " pg_stat_file(fn.filename, true) AS this\n" + " WHERE this.size IS NOT NULL\n" " UNION ALL\n" " SELECT parent.path || parent.filename || '/' AS path,\n" " fn, this.size, this.isdir\n" " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" - " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" + " pg_ls_dir(parent.path || parent.filename, true, true) AS fn,\n" + " pg_stat_file(parent.path || parent.filename || '/' || fn, true)\n" + " AS this\n" + " WHERE parent.isdir = 't' AND\n" + " (fn NOT IN ('.', '..')) AND\n" + " this.size IS NOT NULL)\n" + "SELECT path || filename AS file_path, size, isdir,\n" + " pg_tablespace_location(pg_tablespace.oid, true)\n" + " AS link_target\n" "FROM files\n" "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" " AND oid::text = files.filename\n"; @@ -259,8 +265,7 @@ receiveFileChunks(const char *sql) } if (PQgetisnull(res, 0, 0) || - PQgetisnull(res, 0, 1) || - PQgetisnull(res, 0, 2)) + PQgetisnull(res, 0, 1)) { pg_fatal("unexpected NULL result while fetching remote files\n"); } @@ -278,6 +283,20 @@ receiveFileChunks(const char *sql) memcpy(filename, PQgetvalue(res, 0, 0), filenamelen); filename[filenamelen] = '\0'; + /* + * It may be possible that a file has been deleted on remote side after + * creating the file map. In this case simply ignore it and move on. + */ + if (PQgetisnull(res, 0, 2)) + { + pg_log(PG_DEBUG, + "received NULL chunk for file \"%s\", file has been deleted\n", + filename); + pg_free(filename); + PQclear(res); + continue; + } + chunk = PQgetvalue(res, 0, 2); pg_log(PG_DEBUG, "received chunk for file \"%s\", off %d, len %d\n", @@ -445,7 +464,7 @@ libpq_executeFileMap(filemap_t *map) */ sql = "SELECT path, begin, \n" - " pg_read_binary_file(path, begin, len) AS chunk\n" + " pg_read_binary_file(path, begin, len, true) AS chunk\n" "FROM fetchchunks\n"; receiveFileChunks(sql); -- 2.4.4