From c9a9d7f731a7e103f2346d52d3c6deae0c4d7045 Mon Sep 17 00:00:00 2001
From: Antonin Houska <ah@cybertec.at>
Date: Fri, 5 Jul 2019 16:24:02 +0200
Subject: [PATCH 14/17] Teach pg_upgrade to preserve OIDs of relfilenode,
 database and tablespace.

Since these OIDs are used to construct encryption initialization vector, we
need to keep them the same in the new cluster, otherwise relation files need
to be "reencrypted" during the update and the --link mode must be disabled for
encrypted cluster.
---
 src/backend/catalog/heap.c                 | 71 +++++++++++++++++++----
 src/backend/catalog/index.c                |  1 +
 src/backend/commands/dbcommands.c          | 11 ++++
 src/backend/commands/tablespace.c          | 14 ++++-
 src/backend/utils/adt/pg_upgrade_support.c | 55 ++++++++++++++++++
 src/bin/pg_dump/pg_dump.c                  | 90 ++++++++++++++++++++++++++----
 src/bin/pg_dump/pg_dumpall.c               |  3 +
 src/bin/pg_upgrade/Makefile                |  2 +-
 src/bin/pg_upgrade/pg_upgrade.c            | 14 +++--
 src/include/catalog/binary_upgrade.h       |  7 +++
 src/include/catalog/pg_proc.dat            | 20 +++++++
 11 files changed, 257 insertions(+), 31 deletions(-)

diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 3b8c8b193a..0aef214cae 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -91,7 +91,9 @@
 
 /* Potentially set by pg_upgrade_support functions */
 Oid			binary_upgrade_next_heap_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_heap_pg_class_relfilenode = InvalidOid;
 Oid			binary_upgrade_next_toast_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_toast_pg_class_relfilenode = InvalidOid;
 
 static void AddNewRelationTuple(Relation pg_class_desc,
 								Relation new_rel_desc,
@@ -366,17 +368,60 @@ heap_create(const char *relname,
 			break;
 	}
 
-	/*
-	 * Decide whether to create storage. If caller passed a valid relfilenode,
-	 * storage is already created, so don't do it here.  Also don't create it
-	 * for relkinds without physical storage.
-	 */
-	if (!RELKIND_HAS_STORAGE(relkind) || OidIsValid(relfilenode))
-		create_storage = false;
+	/* Decide whether to create storage. */
+	if (!IsBinaryUpgrade)
+	{
+		/*
+		 * If caller passed a valid relfilenode, storage is already created,
+		 * so don't do it here.  Also don't create it for relkinds without
+		 * physical storage.
+		 */
+		if (!RELKIND_HAS_STORAGE(relkind) || OidIsValid(relfilenode))
+			create_storage = false;
+		else
+		{
+			create_storage = true;
+			relfilenode = relid;
+		}
+	}
 	else
 	{
+		/* Override relfilenode? */
+		if (relkind == RELKIND_RELATION || relkind == RELKIND_SEQUENCE ||
+			relkind == RELKIND_MATVIEW)
+		{
+			if (!OidIsValid(binary_upgrade_next_heap_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("relfilenode value not set when in binary upgrade mode")));
+
+			relfilenode = binary_upgrade_next_heap_pg_class_relfilenode;
+			binary_upgrade_next_heap_pg_class_relfilenode = InvalidOid;
+		}
+		else if (relkind == RELKIND_TOASTVALUE)
+		{
+			if (!OidIsValid(binary_upgrade_next_toast_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("toast relfilenode value not set when in binary upgrade mode")));
+			relfilenode = binary_upgrade_next_toast_pg_class_relfilenode;
+			binary_upgrade_next_toast_pg_class_relfilenode = InvalidOid;
+		}
+		else if (relkind == RELKIND_INDEX)
+		{
+			if (!OidIsValid(binary_upgrade_next_index_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("pg_class index relfilenode value not set when in binary upgrade mode")));
+			relfilenode = binary_upgrade_next_index_pg_class_relfilenode;
+			binary_upgrade_next_index_pg_class_relfilenode = InvalidOid;
+		}
+
+		/*
+		 * The storage is needed for the following DDLs, although it will
+		 * eventually be overwritten with that of the old cluster.
+		 */
 		create_storage = true;
-		relfilenode = relid;
 	}
 
 	/*
@@ -1149,9 +1194,6 @@ heap_create_with_catalog(const char *relname,
 
 	/*
 	 * Allocate an OID for the relation, unless we were told what to use.
-	 *
-	 * The OID will be the relfilenode as well, so make sure it doesn't
-	 * collide with either pg_class OIDs or existing physical files.
 	 */
 	if (!OidIsValid(relid))
 	{
@@ -1179,8 +1221,15 @@ heap_create_with_catalog(const char *relname,
 			binary_upgrade_next_toast_pg_class_oid = InvalidOid;
 		}
 		else
+		{
+			/*
+			 * The OID will be the relfilenode as well, so make sure it
+			 * doesn't collide with either pg_class OIDs or existing physical
+			 * files.
+			 */
 			relid = GetNewRelFileNode(reltablespace, pg_class_desc,
 									  relpersistence);
+		}
 	}
 
 	/*
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index bb60b23093..1566d03dcc 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -83,6 +83,7 @@
 
 /* Potentially set by pg_upgrade_support functions */
 Oid			binary_upgrade_next_index_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_index_pg_class_relfilenode = InvalidOid;
 
 /*
  * Pointer-free representation of variables used when reindexing system
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 4ffb6dccf4..692bd40907 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -92,6 +92,7 @@ static void remove_dbtablespaces(Oid db_id);
 static bool check_db_file_conflict(Oid db_id);
 static int	errdetail_busy_db(int notherbackends, int npreparedxacts);
 
+Oid	binary_upgrade_next_pg_database_oid = InvalidOid;
 
 /*
  * CREATE DATABASE
@@ -523,6 +524,16 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 					(errcode(ERRCODE_DUPLICATE_DATABASE),
 					 errmsg("database \"%s\" already exists", dbname)));
 	}
+	else if (IsBinaryUpgrade)
+	{
+		if (!OidIsValid(binary_upgrade_next_pg_database_oid))
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("pg_database OID value not set when in binary upgrade mode")));
+
+		dboid = binary_upgrade_next_pg_database_oid;
+		binary_upgrade_next_pg_database_oid = InvalidOid;
+	}
 	else
 	{
 		do
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 84efb414d8..dcbecc7196 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -89,6 +89,7 @@
 char	   *default_tablespace = NULL;
 char	   *temp_tablespaces = NULL;
 
+Oid	binary_upgrade_next_pg_tablespace_oid = InvalidOid;
 
 static void create_tablespace_directories(const char *location,
 										  const Oid tablespaceoid);
@@ -336,7 +337,18 @@ CreateTableSpace(CreateTableSpaceStmt *stmt)
 
 	MemSet(nulls, false, sizeof(nulls));
 
-	tablespaceoid = GetNewOidWithIndex(rel, TablespaceOidIndexId,
+	if (IsBinaryUpgrade)
+	{
+		if (!OidIsValid(binary_upgrade_next_pg_tablespace_oid))
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("pg_tablespace OID value not set when in binary upgrade mode")));
+
+		tablespaceoid = binary_upgrade_next_pg_tablespace_oid;
+		binary_upgrade_next_pg_tablespace_oid = InvalidOid;
+	}
+	else
+		tablespaceoid = GetNewOidWithIndex(rel, TablespaceOidIndexId,
 									   Anum_pg_tablespace_oid);
 	values[Anum_pg_tablespace_oid - 1] = ObjectIdGetDatum(tablespaceoid);
 	values[Anum_pg_tablespace_spcname - 1] =
diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c
index 99db5ba389..d2e7c0d7eb 100644
--- a/src/backend/utils/adt/pg_upgrade_support.c
+++ b/src/backend/utils/adt/pg_upgrade_support.c
@@ -30,6 +30,28 @@ do {															\
 } while (0)
 
 Datum
+binary_upgrade_set_next_pg_database_oid(PG_FUNCTION_ARGS)
+{
+	Oid			dboid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_pg_database_oid = dboid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
+binary_upgrade_set_next_pg_tablespace_oid(PG_FUNCTION_ARGS)
+{
+	Oid			tbspoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_pg_tablespace_oid = tbspoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_pg_type_oid(PG_FUNCTION_ARGS)
 {
 	Oid			typoid = PG_GETARG_OID(0);
@@ -74,6 +96,17 @@ binary_upgrade_set_next_heap_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_heap_pg_class_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_heap_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_index_pg_class_oid(PG_FUNCTION_ARGS)
 {
 	Oid			reloid = PG_GETARG_OID(0);
@@ -85,6 +118,17 @@ binary_upgrade_set_next_index_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_index_pg_class_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_index_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_toast_pg_class_oid(PG_FUNCTION_ARGS)
 {
 	Oid			reloid = PG_GETARG_OID(0);
@@ -96,6 +140,17 @@ binary_upgrade_set_next_toast_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_toast_pg_class_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_toast_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_pg_enum_oid(PG_FUNCTION_ARGS)
 {
 	Oid			enumoid = PG_GETARG_OID(0);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 806fc78f04..912bfcb8f3 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -48,6 +48,7 @@
 #include "catalog/pg_attribute_d.h"
 #include "catalog/pg_cast_d.h"
 #include "catalog/pg_class_d.h"
+#include "catalog/pg_database.h"
 #include "catalog/pg_default_acl_d.h"
 #include "catalog/pg_largeobject_d.h"
 #include "catalog/pg_largeobject_metadata_d.h"
@@ -280,7 +281,7 @@ static void binary_upgrade_set_type_oids_by_type_oid(Archive *fout,
 													 bool force_array_type);
 static bool binary_upgrade_set_type_oids_by_rel_oid(Archive *fout,
 													PQExpBuffer upgrade_buffer, Oid pg_rel_oid);
-static void binary_upgrade_set_pg_class_oids(Archive *fout,
+static void binary_upgrade_set_pg_class_oids_and_relfilenodes(Archive *fout,
 											 PQExpBuffer upgrade_buffer,
 											 Oid pg_class_oid, bool is_index);
 static void binary_upgrade_extension_member(PQExpBuffer upgrade_buffer,
@@ -2630,6 +2631,7 @@ dumpDatabase(Archive *fout)
 	PQExpBuffer dbQry = createPQExpBuffer();
 	PQExpBuffer delQry = createPQExpBuffer();
 	PQExpBuffer creaQry = createPQExpBuffer();
+	PQExpBuffer setDBIdQry = createPQExpBuffer();
 	PQExpBuffer labelq = createPQExpBuffer();
 	PGconn	   *conn = GetConnection(fout);
 	PGresult   *res;
@@ -2800,6 +2802,37 @@ dumpDatabase(Archive *fout)
 	qdatname = pg_strdup(fmtId(datname));
 
 	/*
+	 * Make sure that pg_upgrade does not change database OID. Don't care
+	 * about "postgres" database, backend will assign it fixed OID
+	 * anyway. ("template1" has fixed OID too but the value 1 should not
+	 * collide with any other OID so backend pays no attention to it.)
+	 */
+	if (dopt->binary_upgrade && dbCatId.oid != PostgresDbOid)
+	{
+		appendPQExpBufferStr(setDBIdQry, "\n-- For binary upgrade, must preserve pg_database oid\n");
+		appendPQExpBuffer(setDBIdQry,
+						  "SELECT pg_catalog.binary_upgrade_set_next_pg_database_oid('%u'::pg_catalog.oid);\n",
+						  dbCatId.oid);
+
+		/*
+		 * Need a separate entry, otherwise the command will be run in the
+		 * same transaction as the CREATE DATABASE command, which is not
+		 * allowed.
+		 */
+		ArchiveEntry(fout,
+					 dbCatId,		/* catalog ID */
+					 dbDumpId,		/* dump ID */
+					 ARCHIVE_OPTS(.tag = datname,
+								  .owner = dba,
+								  .description = "SET_DB_OID",
+								  .section = SECTION_PRE_DATA,
+								  .createStmt = setDBIdQry->data,
+								  .dropStmt = NULL));
+
+		destroyPQExpBuffer(setDBIdQry);
+	}
+
+	/*
 	 * Prepare the CREATE DATABASE command.  We must specify encoding, locale,
 	 * and tablespace since those can't be altered later.  Other DB properties
 	 * are left to the DATABASE PROPERTIES entry, so that they can be applied
@@ -4391,36 +4424,57 @@ binary_upgrade_set_type_oids_by_rel_oid(Archive *fout,
 	return toast_set;
 }
 
+/*
+ * The function handles both oid and relfilenode since they are related to one
+ * another. It'd be inefficient if we ran two similar queries for each
+ * relation instead of just one.
+ */
 static void
-binary_upgrade_set_pg_class_oids(Archive *fout,
-								 PQExpBuffer upgrade_buffer, Oid pg_class_oid,
-								 bool is_index)
+binary_upgrade_set_pg_class_oids_and_relfilenodes(Archive *fout,
+												  PQExpBuffer upgrade_buffer,
+												  Oid pg_class_oid,
+												  bool is_index)
 {
 	PQExpBuffer upgrade_query = createPQExpBuffer();
 	PGresult   *upgrade_res;
+	Oid			pg_class_relfilenode;
 	Oid			pg_class_reltoastrelid;
+	Oid			pg_class_relfilenode_toast;
 	Oid			pg_index_indexrelid;
+	Oid			pg_class_relfilenode_toast_idx;
 
 	appendPQExpBuffer(upgrade_query,
-					  "SELECT c.reltoastrelid, i.indexrelid "
+					  "SELECT c.relfilenode, c.reltoastrelid, ct.relfilenode AS relfilenode_toast, i.indexrelid, cti.relfilenode AS relfilenode_toast_index "
 					  "FROM pg_catalog.pg_class c LEFT JOIN "
 					  "pg_catalog.pg_index i ON (c.reltoastrelid = i.indrelid AND i.indisvalid) "
+					  "LEFT JOIN pg_catalog.pg_class ct ON (c.reltoastrelid = ct.oid) "
+					  "LEFT JOIN pg_catalog.pg_class AS cti ON (i.indexrelid = cti.oid) "
 					  "WHERE c.oid = '%u'::pg_catalog.oid;",
 					  pg_class_oid);
 
 	upgrade_res = ExecuteSqlQueryForSingleRow(fout, upgrade_query->data);
 
+	pg_class_relfilenode = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "relfilenode")));
 	pg_class_reltoastrelid = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "reltoastrelid")));
+	pg_class_relfilenode_toast = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "relfilenode_toast")));
 	pg_index_indexrelid = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "indexrelid")));
+	pg_class_relfilenode_toast_idx = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "relfilenode_toast_index")));
 
 	appendPQExpBufferStr(upgrade_buffer,
-						 "\n-- For binary upgrade, must preserve pg_class oids\n");
+						 "\n-- For binary upgrade, must preserve pg_class oids and relfilenodes\n");
 
 	if (!is_index)
 	{
 		appendPQExpBuffer(upgrade_buffer,
 						  "SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('%u'::pg_catalog.oid);\n",
 						  pg_class_oid);
+
+		/* Not every relation has storage. */
+		if (OidIsValid(pg_class_relfilenode))
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+							  pg_class_relfilenode);
+
 		/* only tables have toast tables, not indexes */
 		if (OidIsValid(pg_class_reltoastrelid))
 		{
@@ -4436,17 +4490,28 @@ binary_upgrade_set_pg_class_oids(Archive *fout,
 			appendPQExpBuffer(upgrade_buffer,
 							  "SELECT pg_catalog.binary_upgrade_set_next_toast_pg_class_oid('%u'::pg_catalog.oid);\n",
 							  pg_class_reltoastrelid);
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_toast_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+							  pg_class_relfilenode_toast);
 
 			/* every toast table has an index */
 			appendPQExpBuffer(upgrade_buffer,
 							  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('%u'::pg_catalog.oid);\n",
 							  pg_index_indexrelid);
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+							  pg_class_relfilenode_toast_idx);
 		}
 	}
 	else
+	{
 		appendPQExpBuffer(upgrade_buffer,
 						  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('%u'::pg_catalog.oid);\n",
 						  pg_class_oid);
+		appendPQExpBuffer(upgrade_buffer,
+						  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+						  pg_class_relfilenode);
+	}
 
 	appendPQExpBufferChar(upgrade_buffer, '\n');
 
@@ -11008,7 +11073,7 @@ dumpCompositeType(Archive *fout, TypeInfo *tyinfo)
 		binary_upgrade_set_type_oids_by_type_oid(fout, q,
 												 tyinfo->dobj.catId.oid,
 												 false);
-		binary_upgrade_set_pg_class_oids(fout, q, tyinfo->typrelid, false);
+		binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q, tyinfo->typrelid, false);
 	}
 
 	qtypname = pg_strdup(fmtId(tyinfo->dobj.name));
@@ -15507,7 +15572,7 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo)
 		appendPQExpBuffer(delq, "DROP VIEW %s;\n", qualrelname);
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
 											 tbinfo->dobj.catId.oid, false);
 
 		appendPQExpBuffer(q, "CREATE VIEW %s", qualrelname);
@@ -15583,7 +15648,7 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo)
 		appendPQExpBuffer(delq, "DROP %s %s;\n", reltypename, qualrelname);
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
 											 tbinfo->dobj.catId.oid, false);
 
 		appendPQExpBuffer(q, "CREATE %s%s %s",
@@ -16337,7 +16402,7 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo)
 		int			nstatvals;
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
 											 indxinfo->dobj.catId.oid, true);
 
 		/* Plain secondary index */
@@ -16559,7 +16624,7 @@ dumpConstraint(Archive *fout, ConstraintInfo *coninfo)
 				  coninfo->dobj.name);
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
 											 indxinfo->dobj.catId.oid, true);
 
 		appendPQExpBuffer(q, "ALTER TABLE ONLY %s\n",
@@ -16971,12 +17036,13 @@ dumpSequence(Archive *fout, TableInfo *tbinfo)
 
 	if (dopt->binary_upgrade)
 	{
-		binary_upgrade_set_pg_class_oids(fout, query,
+		binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, query,
 										 tbinfo->dobj.catId.oid, false);
 		binary_upgrade_set_type_oids_by_rel_oid(fout, query,
 												tbinfo->dobj.catId.oid);
 	}
 
+
 	if (tbinfo->is_identity_sequence)
 	{
 		TableInfo  *owning_tab = findTableByOid(tbinfo->owning_tab);
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 158c0c74b2..d072f28956 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -1259,6 +1259,9 @@ dumpTablespaces(PGconn *conn)
 		/* needed for buildACLCommands() */
 		fspcname = pg_strdup(fmtId(spcname));
 
+		appendPQExpBufferStr(buf, "\n-- For binary upgrade, must preserve pg_tablespace oid\n");
+		appendPQExpBuffer(buf, "SELECT pg_catalog.binary_upgrade_set_next_pg_tablespace_oid('%u'::pg_catalog.oid);\n", spcoid);
+
 		appendPQExpBuffer(buf, "CREATE TABLESPACE %s", fspcname);
 		appendPQExpBuffer(buf, " OWNER %s", fmtId(spcowner));
 
diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile
index 2cb829acd5..0eaf869119 100644
--- a/src/bin/pg_upgrade/Makefile
+++ b/src/bin/pg_upgrade/Makefile
@@ -11,7 +11,7 @@ OBJS = check.o controldata.o dump.o exec.o file.o function.o info.o \
        option.o parallel.o pg_upgrade.o relfilenode.o server.o \
        tablespace.o util.o version.o $(WIN32RES)
 
-override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS)
+override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 all: pg_upgrade
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index ff78929707..336df7378f 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -15,12 +15,14 @@
  *	oids are the same between old and new clusters.  This is important
  *	because toast oids are stored as toast pointers in user tables.
  *
- *	While pg_class.oid and pg_class.relfilenode are initially the same
- *	in a cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM
- *	FULL.  In the new cluster, pg_class.oid and pg_class.relfilenode will
- *	be the same and will match the old pg_class.oid value.  Because of
- *	this, old/new pg_class.relfilenode values will not match if CLUSTER,
- *	REINDEX, or VACUUM FULL have been performed in the old cluster.
+ *	While pg_class.oid and pg_class.relfilenode are initially the same in a
+ *	cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM FULL.  We
+ *	control assignments of pg_class.relfilenode because relfilenode is used to
+ *	derive initialization vector for data encryption. If relfilenode changed
+ *	during upgrade, we'd be unable to decrypt the data in the new cluster.
+ *
+ *	pg_database.oid and pg_tablespace.oid are also preserved because they
+ *	participate in the encryption initialization vector.
  *
  *	We control all assignments of pg_type.oid because these oids are stored
  *	in user composite type values.
diff --git a/src/include/catalog/binary_upgrade.h b/src/include/catalog/binary_upgrade.h
index 2927b7a4d3..cffafbc219 100644
--- a/src/include/catalog/binary_upgrade.h
+++ b/src/include/catalog/binary_upgrade.h
@@ -14,14 +14,21 @@
 #ifndef BINARY_UPGRADE_H
 #define BINARY_UPGRADE_H
 
+extern PGDLLIMPORT Oid binary_upgrade_next_pg_database_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_pg_tablespace_oid;
+
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_array_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_type_oid;
 
 extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_index_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_index_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_relfilenode;
 
+extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_enum_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_authid_oid;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 29f0944774..bcb78168d3 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10080,6 +10080,26 @@
   proname => 'binary_upgrade_set_missing_value', provolatile => 'v',
   proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text',
   prosrc => 'binary_upgrade_set_missing_value' },
+{ oid => '4142', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_heap_pg_class_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_heap_pg_class_relfilenode' },
+{ oid => '4191', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_index_pg_class_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_index_pg_class_relfilenode' },
+{ oid => '4192', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_toast_pg_class_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_toast_pg_class_relfilenode' },
+{ oid => '4193', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_pg_database_oid', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_pg_database_oid' },
+{ oid => '4194', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_pg_tablespace_oid', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_pg_tablespace_oid' }
 
 # conversion functions
 { oid => '4300',
-- 
2.13.7

