From 60395b366c399b331a8547525188697a40010bef Mon Sep 17 00:00:00 2001
From: shruthikc-gowda <shruthi.kc@enterprisedb.com>
Date: Wed, 11 Aug 2021 13:04:52 +0530
Subject: [PATCH v1] Teach pg_upgrade to preserve OIDs of relfilenode, database
 and tablespace.

Since these OIDs are used to construct encryption initialization vector, we
need to keep them the same in the new cluster, otherwise relation files need
to be "reencrypted" during the update and the --link mode must be disabled for
encrypted cluster.
---
 src/backend/catalog/heap.c                         |  66 ++++++--
 src/backend/catalog/index.c                        |   1 +
 src/backend/commands/dbcommands.c                  |  22 ++-
 src/backend/commands/tablespace.c                  |  14 +-
 src/backend/utils/adt/pg_upgrade_support.c         |  55 +++++++
 src/bin/pg_dump/pg_dump.c                          | 172 ++++++++++++++-------
 src/bin/pg_dump/pg_dumpall.c                       |   3 +
 src/bin/pg_upgrade/Makefile                        |   2 +-
 src/bin/pg_upgrade/dump.c                          |   1 -
 src/bin/pg_upgrade/pg_upgrade.c                    |  14 +-
 src/include/catalog/binary_upgrade.h               |   7 +
 src/include/catalog/pg_proc.dat                    |  20 +++
 .../spgist_name_ops/expected/spgist_name_ops.out   |  14 +-
 13 files changed, 315 insertions(+), 76 deletions(-)

diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 83746d3..483665d 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -91,7 +91,9 @@
 
 /* Potentially set by pg_upgrade_support functions */
 Oid			binary_upgrade_next_heap_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_heap_pg_class_relfilenode = InvalidOid;
 Oid			binary_upgrade_next_toast_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_toast_pg_class_relfilenode = InvalidOid;
 
 static void AddNewRelationTuple(Relation pg_class_desc,
 								Relation new_rel_desc,
@@ -366,17 +368,63 @@ heap_create(const char *relname,
 			break;
 	}
 
-	/*
-	 * Decide whether to create storage. If caller passed a valid relfilenode,
-	 * storage is already created, so don't do it here.  Also don't create it
-	 * for relkinds without physical storage.
-	 */
-	if (!RELKIND_HAS_STORAGE(relkind) || OidIsValid(relfilenode))
-		create_storage = false;
+	/* Decide whether to create storage. */
+	if (!IsBinaryUpgrade)
+	{
+		/*
+		 * If caller passed a valid relfilenode, storage is already created,
+		 * so don't do it here.  Also don't create it for relkinds without
+		 * physical storage.
+		 */
+		if (!RELKIND_HAS_STORAGE(relkind) || OidIsValid(relfilenode))
+			create_storage = false;
+		else
+		{
+			create_storage = true;
+			relfilenode = relid;
+		}
+	}
 	else
 	{
-		create_storage = true;
-		relfilenode = relid;
+		/* Override relfilenode? */
+		if (relkind == RELKIND_RELATION || relkind == RELKIND_SEQUENCE ||
+			relkind == RELKIND_MATVIEW)
+		{
+			if (!OidIsValid(binary_upgrade_next_heap_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("relfilenode value not set when in binary upgrade mode")));
+
+			relfilenode = binary_upgrade_next_heap_pg_class_relfilenode;
+			binary_upgrade_next_heap_pg_class_relfilenode = InvalidOid;
+		}
+		else if (relkind == RELKIND_TOASTVALUE)
+		{
+			if (!OidIsValid(binary_upgrade_next_toast_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("toast relfilenode value not set when in binary upgrade mode")));
+			relfilenode = binary_upgrade_next_toast_pg_class_relfilenode;
+			binary_upgrade_next_toast_pg_class_relfilenode = InvalidOid;
+		}
+		else if (relkind == RELKIND_INDEX)
+		{
+			if (!OidIsValid(binary_upgrade_next_index_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("pg_class index relfilenode value not set when in binary upgrade mode")));
+			relfilenode = binary_upgrade_next_index_pg_class_relfilenode;
+			binary_upgrade_next_index_pg_class_relfilenode = InvalidOid;
+		}
+
+		/*
+		 * The storage is needed for the following DDLs, although it will
+		 * eventually be overwritten with that of the old cluster.
+		 */
+		if (RELKIND_HAS_STORAGE(relkind))
+			create_storage = true;
+		else
+			create_storage = false;
 	}
 
 	/*
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 26bfa74..a124617 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -86,6 +86,7 @@
 
 /* Potentially set by pg_upgrade_support functions */
 Oid			binary_upgrade_next_index_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_index_pg_class_relfilenode = InvalidOid;
 
 /*
  * Pointer-free representation of variables used when reindexing system
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 029fab4..1022ef9 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -92,6 +92,7 @@ static void remove_dbtablespaces(Oid db_id);
 static bool check_db_file_conflict(Oid db_id);
 static int	errdetail_busy_db(int notherbackends, int npreparedxacts);
 
+Oid	binary_upgrade_next_pg_database_oid = InvalidOid;
 
 /*
  * CREATE DATABASE
@@ -504,11 +505,24 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 	 */
 	pg_database_rel = table_open(DatabaseRelationId, RowExclusiveLock);
 
-	do
+	if (IsBinaryUpgrade)
 	{
-		dboid = GetNewOidWithIndex(pg_database_rel, DatabaseOidIndexId,
-								   Anum_pg_database_oid);
-	} while (check_db_file_conflict(dboid));
+		if (!OidIsValid(binary_upgrade_next_pg_database_oid))
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("pg_database OID value not set when in binary upgrade mode")));
+
+		dboid = binary_upgrade_next_pg_database_oid;
+		binary_upgrade_next_pg_database_oid = InvalidOid;
+	}
+	else
+	{
+		do
+		{
+			dboid = GetNewOidWithIndex(pg_database_rel, DatabaseOidIndexId,
+									   Anum_pg_database_oid);
+		} while (check_db_file_conflict(dboid));
+	}
 
 	/*
 	 * Insert a new tuple into pg_database.  This establishes our ownership of
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index a54239a..89f5fea 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -88,6 +88,7 @@
 char	   *default_tablespace = NULL;
 char	   *temp_tablespaces = NULL;
 
+Oid	binary_upgrade_next_pg_tablespace_oid = InvalidOid;
 
 static void create_tablespace_directories(const char *location,
 										  const Oid tablespaceoid);
@@ -335,7 +336,18 @@ CreateTableSpace(CreateTableSpaceStmt *stmt)
 
 	MemSet(nulls, false, sizeof(nulls));
 
-	tablespaceoid = GetNewOidWithIndex(rel, TablespaceOidIndexId,
+	if (IsBinaryUpgrade)
+	{
+		if (!OidIsValid(binary_upgrade_next_pg_tablespace_oid))
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("pg_tablespace OID value not set when in binary upgrade mode")));
+
+		tablespaceoid = binary_upgrade_next_pg_tablespace_oid;
+		binary_upgrade_next_pg_tablespace_oid = InvalidOid;
+	}
+	else
+		tablespaceoid = GetNewOidWithIndex(rel, TablespaceOidIndexId,
 									   Anum_pg_tablespace_oid);
 	values[Anum_pg_tablespace_oid - 1] = ObjectIdGetDatum(tablespaceoid);
 	values[Anum_pg_tablespace_spcname - 1] =
diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c
index b5b46d7..bca67bb 100644
--- a/src/backend/utils/adt/pg_upgrade_support.c
+++ b/src/backend/utils/adt/pg_upgrade_support.c
@@ -30,6 +30,28 @@ do {															\
 } while (0)
 
 Datum
+binary_upgrade_set_next_pg_database_oid(PG_FUNCTION_ARGS)
+{
+	Oid			dboid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_pg_database_oid = dboid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
+binary_upgrade_set_next_pg_tablespace_oid(PG_FUNCTION_ARGS)
+{
+	Oid			tbspoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_pg_tablespace_oid = tbspoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_pg_type_oid(PG_FUNCTION_ARGS)
 {
 	Oid			typoid = PG_GETARG_OID(0);
@@ -85,6 +107,17 @@ binary_upgrade_set_next_heap_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_heap_pg_class_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_heap_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_index_pg_class_oid(PG_FUNCTION_ARGS)
 {
 	Oid			reloid = PG_GETARG_OID(0);
@@ -96,6 +129,17 @@ binary_upgrade_set_next_index_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_index_pg_class_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_index_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_toast_pg_class_oid(PG_FUNCTION_ARGS)
 {
 	Oid			reloid = PG_GETARG_OID(0);
@@ -107,6 +151,17 @@ binary_upgrade_set_next_toast_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_toast_pg_class_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_toast_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_pg_enum_oid(PG_FUNCTION_ARGS)
 {
 	Oid			enumoid = PG_GETARG_OID(0);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 90ac445..56a9ac5 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -47,6 +47,7 @@
 #include "catalog/pg_authid_d.h"
 #include "catalog/pg_cast_d.h"
 #include "catalog/pg_class_d.h"
+#include "catalog/pg_database.h"
 #include "catalog/pg_default_acl_d.h"
 #include "catalog/pg_largeobject_d.h"
 #include "catalog/pg_largeobject_metadata_d.h"
@@ -292,9 +293,9 @@ static void binary_upgrade_set_type_oids_by_type_oid(Archive *fout,
 													 bool include_multirange_type);
 static void binary_upgrade_set_type_oids_by_rel_oid(Archive *fout,
 													PQExpBuffer upgrade_buffer, Oid pg_rel_oid);
-static void binary_upgrade_set_pg_class_oids(Archive *fout,
-											 PQExpBuffer upgrade_buffer,
-											 Oid pg_class_oid, bool is_index);
+static void binary_upgrade_set_pg_class_oids_and_relfilenodes(Archive *fout,
+															  PQExpBuffer upgrade_buffer,
+															  Oid pg_class_oid, bool is_index);
 static void binary_upgrade_extension_member(PQExpBuffer upgrade_buffer,
 											const DumpableObject *dobj,
 											const char *objtype,
@@ -2781,6 +2782,7 @@ dumpDatabase(Archive *fout)
 	PQExpBuffer dbQry = createPQExpBuffer();
 	PQExpBuffer delQry = createPQExpBuffer();
 	PQExpBuffer creaQry = createPQExpBuffer();
+	PQExpBuffer setDBIdQry = createPQExpBuffer();
 	PQExpBuffer labelq = createPQExpBuffer();
 	PGconn	   *conn = GetConnection(fout);
 	PGresult   *res;
@@ -2951,6 +2953,39 @@ dumpDatabase(Archive *fout)
 	qdatname = pg_strdup(fmtId(datname));
 
 	/*
+	 * Make sure that pg_upgrade does not change database OID. Don't care
+	 * about "postgres" database, backend will assign it fixed OID anyway.
+	 * ("template1" has fixed OID too but the value 1 should not collide with
+	 * any other OID so backend pays no attention to it.)
+	 */
+	if (dopt->binary_upgrade)
+	{
+		appendPQExpBufferStr(setDBIdQry, "\n-- For binary upgrade, must preserve pg_database oid\n");
+		appendPQExpBuffer(setDBIdQry,
+						  "SELECT pg_catalog.binary_upgrade_set_next_pg_database_oid('%u'::pg_catalog.oid);\n",
+						  dbCatId.oid);
+
+		dbDumpId = createDumpId();
+
+		/*
+		 * Need a separate entry, otherwise the command will be run in the
+		 * same transaction as the CREATE DATABASE command, which is not
+		 * allowed.
+		 */
+		ArchiveEntry(fout,
+					 dbCatId,	/* catalog ID */
+					 dbDumpId,	/* dump ID */
+					 ARCHIVE_OPTS(.tag = datname,
+								  .owner = dba,
+								  .description = "SET_DB_OID",
+								  .section = SECTION_PRE_DATA,
+								  .createStmt = setDBIdQry->data,
+								  .dropStmt = NULL));
+
+		destroyPQExpBuffer(setDBIdQry);
+	}
+
+	/*
 	 * Prepare the CREATE DATABASE command.  We must specify encoding, locale,
 	 * and tablespace since those can't be altered later.  Other DB properties
 	 * are left to the DATABASE PROPERTIES entry, so that they can be applied
@@ -3000,7 +3035,6 @@ dumpDatabase(Archive *fout)
 					  qdatname);
 
 	dbDumpId = createDumpId();
-
 	ArchiveEntry(fout,
 				 dbCatId,		/* catalog ID */
 				 dbDumpId,		/* dump ID */
@@ -4695,53 +4729,76 @@ binary_upgrade_set_type_oids_by_rel_oid(Archive *fout,
 	destroyPQExpBuffer(upgrade_query);
 }
 
+/*
+ * The function handles both oid and relfilenode since they are related to one
+ * another. It'd be inefficient if we ran two similar queries for each
+ * relation instead of just one.
+ */
 static void
-binary_upgrade_set_pg_class_oids(Archive *fout,
-								 PQExpBuffer upgrade_buffer, Oid pg_class_oid,
-								 bool is_index)
+binary_upgrade_set_pg_class_oids_and_relfilenodes(Archive *fout,
+												  PQExpBuffer upgrade_buffer,
+												  Oid pg_class_oid,
+												  bool is_index)
 {
+	PQExpBuffer upgrade_query = createPQExpBuffer();
+	PGresult   *upgrade_res;
+	Oid			pg_class_relfilenode;
+	Oid			pg_class_reltoastrelid;
+	Oid			pg_class_relfilenode_toast;
+	char		pg_class_relkind;
+	Oid			pg_index_indexrelid;
+	Oid			pg_class_relfilenode_toast_idx;
+
+	/*
+	 * Preserve the OIDs of the table's toast table and index, if any.
+	 * Indexes cannot have toast tables, so we need not make this probe in
+	 * the index code path.
+	 *
+	 * One complexity is that the current table definition might not
+	 * require the creation of a TOAST table, but the old database might
+	 * have a TOAST table that was created earlier, before some wide
+	 * columns were dropped.  By setting the TOAST oid we force creation
+	 * of the TOAST heap and index by the new backend, so we can copy the
+	 * files during binary upgrade without worrying about this case.
+	 */
+	appendPQExpBuffer(upgrade_query,
+					  "SELECT c.relfilenode, c.relkind, c.reltoastrelid, ct.relfilenode AS relfilenode_toast, i.indexrelid, cti.relfilenode AS relfilenode_toast_index "
+					  "FROM pg_catalog.pg_class c LEFT JOIN "
+					  "pg_catalog.pg_index i ON (c.reltoastrelid = i.indrelid AND i.indisvalid) "
+					  "LEFT JOIN pg_catalog.pg_class ct ON (c.reltoastrelid = ct.oid) "
+					  "LEFT JOIN pg_catalog.pg_class AS cti ON (i.indexrelid = cti.oid) "
+					  "WHERE c.oid = '%u'::pg_catalog.oid;",
+					  pg_class_oid);
+
+	upgrade_res = ExecuteSqlQueryForSingleRow(fout, upgrade_query->data);
+
+	pg_class_relfilenode = atooid(PQgetvalue(upgrade_res, 0,
+											 PQfnumber(upgrade_res, "relfilenode")));
+	pg_class_reltoastrelid = atooid(PQgetvalue(upgrade_res, 0,
+											   PQfnumber(upgrade_res, "reltoastrelid")));
+	pg_class_relfilenode_toast = atooid(PQgetvalue(upgrade_res, 0,
+												   PQfnumber(upgrade_res, "relfilenode_toast")));
+	pg_class_relkind = *PQgetvalue(upgrade_res, 0,
+								   PQfnumber(upgrade_res, "relkind"));
+	pg_index_indexrelid = atooid(PQgetvalue(upgrade_res, 0,
+											PQfnumber(upgrade_res, "indexrelid")));
+	pg_class_relfilenode_toast_idx = atooid(PQgetvalue(upgrade_res, 0,
+													   PQfnumber(upgrade_res, "relfilenode_toast_index")));
+
 	appendPQExpBufferStr(upgrade_buffer,
-						 "\n-- For binary upgrade, must preserve pg_class oids\n");
+						 "\n-- For binary upgrade, must preserve pg_class oids and relfilenodes\n");
 
 	if (!is_index)
 	{
-		PQExpBuffer upgrade_query = createPQExpBuffer();
-		PGresult   *upgrade_res;
-		Oid			pg_class_reltoastrelid;
-		char		pg_class_relkind;
-		Oid			pg_index_indexrelid;
-
 		appendPQExpBuffer(upgrade_buffer,
 						  "SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('%u'::pg_catalog.oid);\n",
 						  pg_class_oid);
 
-		/*
-		 * Preserve the OIDs of the table's toast table and index, if any.
-		 * Indexes cannot have toast tables, so we need not make this probe in
-		 * the index code path.
-		 *
-		 * One complexity is that the current table definition might not
-		 * require the creation of a TOAST table, but the old database might
-		 * have a TOAST table that was created earlier, before some wide
-		 * columns were dropped.  By setting the TOAST oid we force creation
-		 * of the TOAST heap and index by the new backend, so we can copy the
-		 * files during binary upgrade without worrying about this case.
-		 */
-		appendPQExpBuffer(upgrade_query,
-						  "SELECT c.reltoastrelid, c.relkind, i.indexrelid "
-						  "FROM pg_catalog.pg_class c LEFT JOIN "
-						  "pg_catalog.pg_index i ON (c.reltoastrelid = i.indrelid AND i.indisvalid) "
-						  "WHERE c.oid = '%u'::pg_catalog.oid;",
-						  pg_class_oid);
-
-		upgrade_res = ExecuteSqlQueryForSingleRow(fout, upgrade_query->data);
-
-		pg_class_reltoastrelid = atooid(PQgetvalue(upgrade_res, 0,
-												   PQfnumber(upgrade_res, "reltoastrelid")));
-		pg_class_relkind = *PQgetvalue(upgrade_res, 0,
-									   PQfnumber(upgrade_res, "relkind"));
-		pg_index_indexrelid = atooid(PQgetvalue(upgrade_res, 0,
-												PQfnumber(upgrade_res, "indexrelid")));
+		/* Not every relation has storage. */
+		if (OidIsValid(pg_class_relfilenode))
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+							  pg_class_relfilenode);
 
 		/*
 		 * In a pre-v12 database, partitioned tables might be marked as having
@@ -4753,20 +4810,31 @@ binary_upgrade_set_pg_class_oids(Archive *fout,
 			appendPQExpBuffer(upgrade_buffer,
 							  "SELECT pg_catalog.binary_upgrade_set_next_toast_pg_class_oid('%u'::pg_catalog.oid);\n",
 							  pg_class_reltoastrelid);
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_toast_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+							  pg_class_relfilenode_toast);
 
 			/* every toast table has an index */
 			appendPQExpBuffer(upgrade_buffer,
 							  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('%u'::pg_catalog.oid);\n",
 							  pg_index_indexrelid);
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+							  pg_class_relfilenode_toast_idx);
 		}
 
 		PQclear(upgrade_res);
 		destroyPQExpBuffer(upgrade_query);
 	}
 	else
+	{
 		appendPQExpBuffer(upgrade_buffer,
 						  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('%u'::pg_catalog.oid);\n",
 						  pg_class_oid);
+		appendPQExpBuffer(upgrade_buffer,
+						  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_relfilenode('%u'::pg_catalog.oid);\n",
+						  pg_class_relfilenode);
+	}
 
 	appendPQExpBufferChar(upgrade_buffer, '\n');
 }
@@ -11553,7 +11621,7 @@ dumpCompositeType(Archive *fout, const TypeInfo *tyinfo)
 		binary_upgrade_set_type_oids_by_type_oid(fout, q,
 												 tyinfo->dobj.catId.oid,
 												 false, false);
-		binary_upgrade_set_pg_class_oids(fout, q, tyinfo->typrelid, false);
+		binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q, tyinfo->typrelid, false);
 	}
 
 	qtypname = pg_strdup(fmtId(tyinfo->dobj.name));
@@ -15908,8 +15976,8 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
 		appendPQExpBuffer(delq, "DROP VIEW %s;\n", qualrelname);
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
-											 tbinfo->dobj.catId.oid, false);
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
+															  tbinfo->dobj.catId.oid, false);
 
 		appendPQExpBuffer(q, "CREATE VIEW %s", qualrelname);
 
@@ -15987,8 +16055,8 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
 		appendPQExpBuffer(delq, "DROP %s %s;\n", reltypename, qualrelname);
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
-											 tbinfo->dobj.catId.oid, false);
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
+															  tbinfo->dobj.catId.oid, false);
 
 		appendPQExpBuffer(q, "CREATE %s%s %s",
 						  tbinfo->relpersistence == RELPERSISTENCE_UNLOGGED ?
@@ -16798,8 +16866,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
 		int			nstatvals = 0;
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
-											 indxinfo->dobj.catId.oid, true);
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
+															  indxinfo->dobj.catId.oid, true);
 
 		/* Plain secondary index */
 		appendPQExpBuffer(q, "%s;\n", indxinfo->indexdef);
@@ -17054,8 +17122,8 @@ dumpConstraint(Archive *fout, const ConstraintInfo *coninfo)
 				  coninfo->dobj.name);
 
 		if (dopt->binary_upgrade)
-			binary_upgrade_set_pg_class_oids(fout, q,
-											 indxinfo->dobj.catId.oid, true);
+			binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, q,
+															  indxinfo->dobj.catId.oid, true);
 
 		appendPQExpBuffer(q, "ALTER %sTABLE ONLY %s\n", foreign,
 						  fmtQualifiedDumpable(tbinfo));
@@ -17471,8 +17539,8 @@ dumpSequence(Archive *fout, const TableInfo *tbinfo)
 
 	if (dopt->binary_upgrade)
 	{
-		binary_upgrade_set_pg_class_oids(fout, query,
-										 tbinfo->dobj.catId.oid, false);
+		binary_upgrade_set_pg_class_oids_and_relfilenodes(fout, query,
+														  tbinfo->dobj.catId.oid, false);
 
 		/*
 		 * In older PG versions a sequence will have a pg_type entry, but v14
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index c291017..618187c 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -1265,6 +1265,9 @@ dumpTablespaces(PGconn *conn)
 		/* needed for buildACLCommands() */
 		fspcname = pg_strdup(fmtId(spcname));
 
+		appendPQExpBufferStr(buf, "\n-- For binary upgrade, must preserve pg_tablespace oid\n");
+		appendPQExpBuffer(buf, "SELECT pg_catalog.binary_upgrade_set_next_pg_tablespace_oid('%u'::pg_catalog.oid);\n", spcoid);
+
 		appendPQExpBuffer(buf, "CREATE TABLESPACE %s", fspcname);
 		appendPQExpBuffer(buf, " OWNER %s", fmtId(spcowner));
 
diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile
index 44d06be..e862166 100644
--- a/src/bin/pg_upgrade/Makefile
+++ b/src/bin/pg_upgrade/Makefile
@@ -25,7 +25,7 @@ OBJS = \
 	util.o \
 	version.o
 
-override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS)
+override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 all: pg_upgrade
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 90060d0..5010520 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -49,7 +49,6 @@ generate_old_dump(void)
 		pg_log(PG_STATUS, "%s", old_db->db_name);
 		snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid);
 		snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
-
 		parallel_exec_prog(log_file_name, NULL,
 						   "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
 						   "--binary-upgrade --format=custom %s --file=\"%s\" %s",
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index 3628bd7..d7a9a0b 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -15,12 +15,14 @@
  *	oids are the same between old and new clusters.  This is important
  *	because toast oids are stored as toast pointers in user tables.
  *
- *	While pg_class.oid and pg_class.relfilenode are initially the same
- *	in a cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM
- *	FULL.  In the new cluster, pg_class.oid and pg_class.relfilenode will
- *	be the same and will match the old pg_class.oid value.  Because of
- *	this, old/new pg_class.relfilenode values will not match if CLUSTER,
- *	REINDEX, or VACUUM FULL have been performed in the old cluster.
+ *	While pg_class.oid and pg_class.relfilenode are initially the same in a
+ *	cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM FULL.  We
+ *	control assignments of pg_class.relfilenode because relfilenode is used to
+ *	derive initialization vector for data encryption. If relfilenode changed
+ *	during upgrade, we'd be unable to decrypt the data in the new cluster.
+ *
+ *	pg_database.oid and pg_tablespace.oid are also preserved because they
+ *	participate in the encryption initialization vector.
  *
  *	We control all assignments of pg_type.oid because these oids are stored
  *	in user composite type values.
diff --git a/src/include/catalog/binary_upgrade.h b/src/include/catalog/binary_upgrade.h
index f6e82e7..b74e284 100644
--- a/src/include/catalog/binary_upgrade.h
+++ b/src/include/catalog/binary_upgrade.h
@@ -14,15 +14,22 @@
 #ifndef BINARY_UPGRADE_H
 #define BINARY_UPGRADE_H
 
+extern PGDLLIMPORT Oid binary_upgrade_next_pg_database_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_pg_tablespace_oid;
+
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_array_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_mrng_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_mrng_array_pg_type_oid;
 
 extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_index_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_index_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_relfilenode;
 
+extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_enum_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_authid_oid;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index b603700..6cf948b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11036,6 +11036,26 @@
   proname => 'binary_upgrade_set_missing_value', provolatile => 'v',
   proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text',
   prosrc => 'binary_upgrade_set_missing_value' },
+{ oid => '4544', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_heap_pg_class_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_heap_pg_class_relfilenode' },
+{ oid => '4545', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_index_pg_class_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_index_pg_class_relfilenode' },
+{ oid => '4546', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_toast_pg_class_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_toast_pg_class_relfilenode' },
+{ oid => '4547', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_pg_database_oid', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_pg_database_oid' },
+{ oid => '4548', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_pg_tablespace_oid', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_pg_tablespace_oid' }
 
 # conversion functions
 { oid => '4302',
diff --git a/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out b/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out
index ac0ddce..b644858 100644
--- a/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out
+++ b/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out
@@ -52,14 +52,19 @@ select * from t
 ------------------------------------------------------+----+------------------------------------------------------
  binary_upgrade_set_next_array_pg_type_oid            |    | binary_upgrade_set_next_array_pg_type_oid
  binary_upgrade_set_next_heap_pg_class_oid            |    | binary_upgrade_set_next_heap_pg_class_oid
+ binary_upgrade_set_next_heap_pg_class_relfilenode    |    | binary_upgrade_set_next_heap_pg_class_relfilenode
  binary_upgrade_set_next_index_pg_class_oid           |  1 | binary_upgrade_set_next_index_pg_class_oid
+ binary_upgrade_set_next_index_pg_class_relfilenode   |  1 | binary_upgrade_set_next_index_pg_class_relfilenode
  binary_upgrade_set_next_multirange_array_pg_type_oid |  1 | binary_upgrade_set_next_multirange_array_pg_type_oid
  binary_upgrade_set_next_multirange_pg_type_oid       |  1 | binary_upgrade_set_next_multirange_pg_type_oid
  binary_upgrade_set_next_pg_authid_oid                |    | binary_upgrade_set_next_pg_authid_oid
+ binary_upgrade_set_next_pg_database_oid              |    | binary_upgrade_set_next_pg_database_oid
  binary_upgrade_set_next_pg_enum_oid                  |    | binary_upgrade_set_next_pg_enum_oid
+ binary_upgrade_set_next_pg_tablespace_oid            |    | binary_upgrade_set_next_pg_tablespace_oid
  binary_upgrade_set_next_pg_type_oid                  |    | binary_upgrade_set_next_pg_type_oid
  binary_upgrade_set_next_toast_pg_class_oid           |  1 | binary_upgrade_set_next_toast_pg_class_oid
-(9 rows)
+ binary_upgrade_set_next_toast_pg_class_relfilenode   |  1 | binary_upgrade_set_next_toast_pg_class_relfilenode
+(14 rows)
 
 -- Verify clean failure when INCLUDE'd columns result in overlength tuple
 -- The error message details are platform-dependent, so show only SQLSTATE
@@ -97,14 +102,19 @@ select * from t
 ------------------------------------------------------+----+------------------------------------------------------
  binary_upgrade_set_next_array_pg_type_oid            |    | binary_upgrade_set_next_array_pg_type_oid
  binary_upgrade_set_next_heap_pg_class_oid            |    | binary_upgrade_set_next_heap_pg_class_oid
+ binary_upgrade_set_next_heap_pg_class_relfilenode    |    | binary_upgrade_set_next_heap_pg_class_relfilenode
  binary_upgrade_set_next_index_pg_class_oid           |  1 | binary_upgrade_set_next_index_pg_class_oid
+ binary_upgrade_set_next_index_pg_class_relfilenode   |  1 | binary_upgrade_set_next_index_pg_class_relfilenode
  binary_upgrade_set_next_multirange_array_pg_type_oid |  1 | binary_upgrade_set_next_multirange_array_pg_type_oid
  binary_upgrade_set_next_multirange_pg_type_oid       |  1 | binary_upgrade_set_next_multirange_pg_type_oid
  binary_upgrade_set_next_pg_authid_oid                |    | binary_upgrade_set_next_pg_authid_oid
+ binary_upgrade_set_next_pg_database_oid              |    | binary_upgrade_set_next_pg_database_oid
  binary_upgrade_set_next_pg_enum_oid                  |    | binary_upgrade_set_next_pg_enum_oid
+ binary_upgrade_set_next_pg_tablespace_oid            |    | binary_upgrade_set_next_pg_tablespace_oid
  binary_upgrade_set_next_pg_type_oid                  |    | binary_upgrade_set_next_pg_type_oid
  binary_upgrade_set_next_toast_pg_class_oid           |  1 | binary_upgrade_set_next_toast_pg_class_oid
-(9 rows)
+ binary_upgrade_set_next_toast_pg_class_relfilenode   |  1 | binary_upgrade_set_next_toast_pg_class_relfilenode
+(14 rows)
 
 \set VERBOSITY sqlstate
 insert into t values(repeat('xyzzy', 12), 42, repeat('xyzzy', 4000));
-- 
1.8.3.1

