From fcb38a6568a1f2e359642e7d658192d1a1688a58 Mon Sep 17 00:00:00 2001
From: Rishu Bagga <bagrishu@amazon.com>
Date: Thu, 15 Sep 2022 09:57:58 +0000
Subject: [PATCH 4/4] slru_to_buf_page_headers_with_upgrade

---
 src/backend/access/transam/clog.c      |  46 ++++---
 src/backend/access/transam/commit_ts.c |  60 +++++++--
 src/backend/access/transam/multixact.c |  76 ++++++-----
 src/backend/access/transam/subtrans.c  |  12 +-
 src/backend/commands/async.c           |  40 +++---
 src/backend/storage/buffer/bufmgr.c    |  27 +---
 src/backend/storage/lmgr/predicate.c   |  10 +-
 src/backend/storage/page/bufpage.c     |  23 +++-
 src/bin/pg_upgrade/file.c              | 175 ++++++++++++++++++++++++-
 src/bin/pg_upgrade/function.c          |  66 ++++++++++
 src/bin/pg_upgrade/pg_upgrade.c        |  74 +++++++++--
 src/bin/pg_upgrade/pg_upgrade.h        |  19 ++-
 src/include/access/nreldefs.h          |  19 +++
 src/include/access/slru.h              |   2 +
 src/include/storage/bufmgr.h           |   5 +-
 src/include/storage/bufpage.h          |  17 +++
 16 files changed, 556 insertions(+), 115 deletions(-)
 create mode 100644 src/include/access/nreldefs.h

diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c
index 88a275f5823..67920274171 100644
--- a/src/backend/access/transam/clog.c
+++ b/src/backend/access/transam/clog.c
@@ -63,7 +63,7 @@
 /* We need two bits per xact, so four xacts fit in a byte */
 #define CLOG_BITS_PER_XACT	2
 #define CLOG_XACTS_PER_BYTE 4
-#define CLOG_XACTS_PER_PAGE (BLCKSZ * CLOG_XACTS_PER_BYTE)
+#define CLOG_XACTS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData) * CLOG_XACTS_PER_BYTE)
 #define CLOG_XACT_BITMASK	((1 << CLOG_BITS_PER_XACT) - 1)
 
 #define TransactionIdToPage(xid)	((xid) / (TransactionId) CLOG_XACTS_PER_PAGE)
@@ -88,7 +88,7 @@
 
 static Buffer ZeroCLOGPage(int pageno, bool writeXlog);
 static bool CLOGPagePrecedes(int page1, int page2);
-static void WriteZeroPageXlogRec(int pageno);
+static XLogRecPtr WriteZeroPageXlogRec(int pageno);
 static void WriteTruncateXlogRec(int pageno, TransactionId oldestXact,
 								 Oid oldestXactDb);
 static void TransactionIdSetPageStatus(TransactionId xid, int nsubxids,
@@ -353,6 +353,7 @@ TransactionIdSetPageStatusInternal(TransactionId xid, int nsubxids,
 	 * we think.
 	 */
 	buffer = ReadNrelBuffer(NREL_CLOG_REL_ID, pageno);
+
 	LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 
 	/*
@@ -389,6 +390,7 @@ TransactionIdSetPageStatusInternal(TransactionId xid, int nsubxids,
 		TransactionIdSetStatusBit(subxids[i], status, lsn, buffer);
 	}
 
+	
 	MarkBufferDirty(buffer);
 	UnlockReleaseBuffer(buffer);
 }
@@ -575,7 +577,7 @@ TransactionIdSetStatusBit(TransactionId xid, XidStatus status, XLogRecPtr lsn, B
 	Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(GetBufferDescriptor(buffer - 1)),
 								LW_EXCLUSIVE));
 
-	byteptr = BufferGetPage(buffer) + byteno;
+	byteptr = PageGetContents(BufferGetPage(buffer)) + byteno;
 	curval = (*byteptr >> bshift) & CLOG_XACT_BITMASK;
 
 	/*
@@ -603,6 +605,7 @@ TransactionIdSetStatusBit(TransactionId xid, XidStatus status, XLogRecPtr lsn, B
 	byteval |= (status << bshift);
 	*byteptr = byteval;
 
+
 	/*
 	 * Update the buffer LSN if the transaction completion LSN is higher.
 	 *
@@ -613,8 +616,8 @@ TransactionIdSetStatusBit(TransactionId xid, XidStatus status, XLogRecPtr lsn, B
 	 */
 	if (!XLogRecPtrIsInvalid(lsn))
 	{
-		if (BufferGetExternalLSN(GetBufferDescriptor(buffer)) < lsn)
-			BufferSetExternalLSN(GetBufferDescriptor(buffer), lsn);
+		if (PageGetLSN(BufferGetPage(buffer)) < lsn)
+			PageSetLSN(BufferGetPage(buffer), lsn);
 	}
 }
 
@@ -644,11 +647,11 @@ TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn)
 	Buffer		buffer;
 
 	buffer = ReadNrelBuffer(NREL_CLOG_REL_ID, pageno);
-	byteptr = BufferGetPage(buffer) + byteno;
 
-	status = (*byteptr >> bshift) & CLOG_XACT_BITMASK;
+	byteptr = PageGetContents(BufferGetPage(buffer)) + byteno;
 
-	*lsn = BufferGetExternalLSN(GetBufferDescriptor(buffer));
+	status = (*byteptr >> bshift) & CLOG_XACT_BITMASK;
+	*lsn   = PageGetLSN(BufferGetPage(buffer));
 
 	ReleaseBuffer(buffer);
 
@@ -688,13 +691,20 @@ static Buffer
 ZeroCLOGPage(int pageno, bool writeXlog)
 {
 	Buffer		buffer;
+	Page 		page;
+	XLogRecPtr  lsn;
 
 	buffer = ZeroNrelBuffer(NREL_CLOG_REL_ID, pageno);
+	
+	page = BufferGetPage(buffer);
+	PageInitNREL(page, BLCKSZ, 0);
 
-	MarkBufferDirty(buffer);
-
+	lsn = 0;
 	if (writeXlog)
-		WriteZeroPageXlogRec(pageno);
+		lsn = WriteZeroPageXlogRec(pageno);
+	
+	PageSetHeaderDataNonRel(page, pageno, lsn, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION);
+	MarkBufferDirty(buffer);
 
 	return buffer;
 }
@@ -738,13 +748,16 @@ TrimCLOG(void)
 
 		buffer = ReadNrelBuffer(NREL_CLOG_REL_ID, pageno);
 
+
 		LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
-		byteptr = BufferGetPage(buffer) + byteno;
+		byteptr = PageGetContents(BufferGetPage(buffer)) + byteno;
 
 		/* Zero so-far-unused positions in the current byte */
 		*byteptr &= (1 << bshift) - 1;
 		/* Zero the rest of the page */
-		MemSet(byteptr + 1, 0, BLCKSZ - byteno - 1);
+		MemSet(byteptr + 1, 0, BLCKSZ - byteno - MAXALIGN(SizeOfPageHeaderData) - 1);
+
+
 
 		MarkBufferDirty(buffer);
 
@@ -871,12 +884,15 @@ CLOGPagePrecedes(int page1, int page2)
 /*
  * Write a ZEROPAGE xlog record
  */
-static void
+static XLogRecPtr 
 WriteZeroPageXlogRec(int pageno)
 {
+	XLogRecPtr lsn;
 	XLogBeginInsert();
 	XLogRegisterData((char *) (&pageno), sizeof(int));
-	(void) XLogInsert(RM_CLOG_ID, CLOG_ZEROPAGE);
+	lsn = XLogInsert(RM_CLOG_ID, CLOG_ZEROPAGE);
+
+	return lsn;
 }
 
 /*
diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c
index 6cb2870e5f4..c522c21d6cd 100644
--- a/src/backend/access/transam/commit_ts.c
+++ b/src/backend/access/transam/commit_ts.c
@@ -64,13 +64,15 @@ typedef struct CommitTimestampEntry
 									sizeof(RepOriginId))
 
 #define COMMIT_TS_XACTS_PER_PAGE \
-	(BLCKSZ / SizeOfCommitTimestampEntry)
+	((BLCKSZ - SizeOfPageHeaderData) / SizeOfCommitTimestampEntry)
 
 #define TransactionIdToCTsPage(xid) \
 	((xid) / (TransactionId) COMMIT_TS_XACTS_PER_PAGE)
 #define TransactionIdToCTsEntry(xid)	\
 	((xid) % (TransactionId) COMMIT_TS_XACTS_PER_PAGE)
 
+
+
 /*
  * We keep a cache of the last value set in shared memory.
  *
@@ -105,7 +107,8 @@ static Buffer ZeroCommitTsPage(int pageno, bool writeXlog);
 static bool CommitTsPagePrecedes(int page1, int page2);
 static void ActivateCommitTs(void);
 static void DeactivateCommitTs(void);
-static void WriteZeroPageXlogRec(int pageno);
+static XLogRecPtr WriteZeroPageXlogRec(int pageno);
+
 static void WriteTruncateXlogRec(int pageno, TransactionId oldestXid);
 
 /*
@@ -215,6 +218,7 @@ SetXidCommitTsInPage(TransactionId xid, int nsubxids,
 
 	buffer = ReadNrelBuffer(NREL_COMMITTS_REL_ID, pageno);
 
+
 	LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 
 	TransactionIdSetCommitTs(xid, ts, nodeid, buffer);
@@ -234,6 +238,8 @@ TransactionIdSetCommitTs(TransactionId xid, TimestampTz ts,
 						 RepOriginId nodeid, Buffer buffer)
 {
 	int			entryno = TransactionIdToCTsEntry(xid);
+	int 		pageno  = TransactionIdToCTsPage(xid);
+	
 	CommitTimestampEntry entry;
 
 	Assert(TransactionIdIsNormal(xid));
@@ -241,8 +247,12 @@ TransactionIdSetCommitTs(TransactionId xid, TimestampTz ts,
 	entry.time = ts;
 	entry.nodeid = nodeid;
 
-	memcpy(BufferGetPage(buffer) + SizeOfCommitTimestampEntry * entryno,
+	Assert(xid == pageno * COMMIT_TS_XACTS_PER_PAGE + entryno);
+
+	memcpy(PageGetContents(BufferGetPage(buffer)) + \
+		   SizeOfCommitTimestampEntry * entryno,
 		   &entry, SizeOfCommitTimestampEntry);
+
 }
 
 /*
@@ -318,12 +328,13 @@ TransactionIdGetCommitTsData(TransactionId xid, TimestampTz *ts,
 
 	buffer = ReadNrelBuffer(NREL_COMMITTS_REL_ID, pageno);
 
+
 	LockBuffer(buffer, BUFFER_LOCK_SHARE);
 
 	memcpy(&entry,
-		   BufferGetPage(buffer) +
-		   SizeOfCommitTimestampEntry * entryno,
-		   SizeOfCommitTimestampEntry);
+			PageGetContents(BufferGetPage(buffer)) + \
+			SizeOfCommitTimestampEntry * entryno,
+			SizeOfCommitTimestampEntry);
 
 	*ts = entry.time;
 	if (nodeid)
@@ -546,14 +557,22 @@ static Buffer
 ZeroCommitTsPage(int pageno, bool writeXlog)
 {
 	Buffer		buffer;
+	Page 		page;
+	XLogRecPtr  lsn;
 
 	buffer = ZeroNrelBuffer(NREL_COMMITTS_REL_ID, pageno);
-
-	MarkBufferDirty(buffer);
-
+	
+	page = BufferGetPage(buffer);
+	PageInitNREL(page, BLCKSZ, 0);
+	
 	if (writeXlog)
-		WriteZeroPageXlogRec(pageno);
-
+	{
+		lsn = WriteZeroPageXlogRec(pageno);
+		page = BufferGetPage(buffer);
+		PageSetHeaderDataNonRel(page, pageno, lsn, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION);
+	}
+	
+	MarkBufferDirty(buffer);
 	return buffer;
 }
 
@@ -677,8 +696,13 @@ ActivateCommitTs(void)
 	if (!NonRelDoesPhysicalPageExist(NREL_COMMITTS_REL_ID, pageno))
 	{
 		Buffer		buffer;
+		Page 		page;
+
 
 		buffer = ZeroNrelBuffer(NREL_COMMITTS_REL_ID, pageno);
+		
+		page = BufferGetPage(buffer);
+		PageInitNREL(page, BLCKSZ, 0);
 
 		MarkBufferDirty(buffer);
 		FlushOneBuffer(buffer);
@@ -889,12 +913,16 @@ CommitTsPagePrecedes(int page1, int page2)
 /*
  * Write a ZEROPAGE xlog record
  */
-static void
+static XLogRecPtr 
 WriteZeroPageXlogRec(int pageno)
 {
+	XLogRecPtr lsn;
+	
 	XLogBeginInsert();
 	XLogRegisterData((char *) (&pageno), sizeof(int));
-	(void) XLogInsert(RM_COMMIT_TS_ID, COMMIT_TS_ZEROPAGE);
+	lsn = XLogInsert(RM_COMMIT_TS_ID, COMMIT_TS_ZEROPAGE);
+	
+	return lsn;
 }
 
 /*
@@ -928,11 +956,15 @@ commit_ts_redo(XLogReaderState *record)
 	{
 		int			pageno;
 		Buffer		buffer;
+		Page		page;
 
 		memcpy(&pageno, XLogRecGetData(record), sizeof(int));
 
 		buffer = ZeroNrelBuffer(NREL_COMMITTS_REL_ID, pageno);
-
+		
+		page = BufferGetPage(buffer);
+		PageInitNREL(page, BLCKSZ, 0);
+		
 		MarkBufferDirty(buffer);
 		FlushOneBuffer(buffer);
 		UnlockReleaseBuffer(buffer);
diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c
index 810aec99a49..5b87faa8e42 100644
--- a/src/backend/access/transam/multixact.c
+++ b/src/backend/access/transam/multixact.c
@@ -107,7 +107,7 @@
  */
 
 /* We need four bytes per offset */
-#define MULTIXACT_OFFSETS_PER_PAGE (BLCKSZ / sizeof(MultiXactOffset))
+#define MULTIXACT_OFFSETS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData)/ sizeof(MultiXactOffset))
 
 #define MultiXactIdToOffsetPage(xid) \
 	((xid) / (MultiXactOffset) MULTIXACT_OFFSETS_PER_PAGE)
@@ -139,7 +139,7 @@
 /* size in bytes of a complete group */
 #define MULTIXACT_MEMBERGROUP_SIZE \
 	(sizeof(TransactionId) * MULTIXACT_MEMBERS_PER_MEMBERGROUP + MULTIXACT_FLAGBYTES_PER_GROUP)
-#define MULTIXACT_MEMBERGROUPS_PER_PAGE (BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE)
+#define MULTIXACT_MEMBERGROUPS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData) / MULTIXACT_MEMBERGROUP_SIZE)
 #define MULTIXACT_MEMBERS_PER_PAGE	\
 	(MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP)
 
@@ -162,9 +162,9 @@
 
 /* Location (byte offset within page) of flag word for a given member */
 #define MXOffsetToFlagsOffset(xid) \
-	((((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) % \
+	(((((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) % \
 	  (TransactionId) MULTIXACT_MEMBERGROUPS_PER_PAGE) * \
-	 (TransactionId) MULTIXACT_MEMBERGROUP_SIZE)
+	 (TransactionId) MULTIXACT_MEMBERGROUP_SIZE))
 #define MXOffsetToFlagsBitShift(xid) \
 	(((xid) % (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) * \
 	 MXACT_MEMBER_BITS_PER_XACT)
@@ -357,7 +357,7 @@ static bool MultiXactOffsetWouldWrap(MultiXactOffset boundary,
 									 MultiXactOffset start, uint32 distance);
 static bool SetOffsetVacuumLimit(bool is_startup);
 static bool find_multixact_start(MultiXactId multi, MultiXactOffset *result);
-static void WriteMZeroPageXlogRec(int pageno, uint8 info);
+static XLogRecPtr WriteMZeroPageXlogRec(int pageno, uint8 info);
 static void WriteMTruncateXlogRec(Oid oldestMultiDB,
 								  MultiXactId startTruncOff,
 								  MultiXactId endTruncOff,
@@ -876,10 +876,9 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset,
 	else 
 		offset_buf = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
 
-
 	LockBuffer(offset_buf, BUFFER_LOCK_EXCLUSIVE);
 						
-	offptr = (MultiXactOffset *) BufferGetPage(offset_buf);
+	offptr = (MultiXactOffset *) PageGetContents(BufferGetPage(offset_buf));
 	offptr += entryno;
 
 	*offptr = offset;
@@ -921,16 +920,15 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset,
 			else
 				buffer = ReadNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno);
 
-
 			LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 			prev_pageno = pageno;
 		}
 
-		memberptr = (TransactionId *) (BufferGetPage(buffer) + memberoff);
+		memberptr = (TransactionId *) (PageGetContents(BufferGetPage(buffer)) + memberoff);
 
 		*memberptr = members[i].xid;
 
-		flagsptr = (uint32 *) (BufferGetPage(buffer) + flagsoff);
+		flagsptr = (uint32 *) (PageGetContents(BufferGetPage(buffer)) + flagsoff);
 
 		flagsval = *flagsptr;
 		flagsval &= ~(((1 << MXACT_MEMBER_BITS_PER_XACT) - 1) << bshift);
@@ -1369,7 +1367,7 @@ retry:
 	buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
 
 	LockBuffer(buffer, BUFFER_LOCK_SHARE);
-	offptr = (MultiXactOffset *) BufferGetPage(buffer);
+	offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer)));
 	offptr += entryno;
 	offset = *offptr;
 
@@ -1404,10 +1402,11 @@ retry:
 			UnlockReleaseBuffer(buffer);
 			buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
 
+
 			LockBuffer(buffer, BUFFER_LOCK_SHARE);
 		}
 
-		offptr = (MultiXactOffset *) BufferGetPage(buffer);
+		offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer)));
 		offptr += entryno;
 		nextMXOffset = *offptr;
 
@@ -1450,7 +1449,7 @@ retry:
 			prev_pageno = pageno;
 		}
 
-		xactptr = (TransactionId *) (BufferGetPage(buffer) + memberoff);
+		xactptr = (TransactionId *) (PageGetContents((BufferGetPage(buffer)) + memberoff));
 
 		if (!TransactionIdIsValid(*xactptr))
 		{
@@ -1461,7 +1460,7 @@ retry:
 
 		flagsoff = MXOffsetToFlagsOffset(offset);
 		bshift = MXOffsetToFlagsBitShift(offset);
-		flagsptr = (uint32 *) (BufferGetPage(buffer) + flagsoff);
+		flagsptr = (uint32 *) (PageGetContents(BufferGetPage(buffer)) + flagsoff);
 
 		ptr[truelength].xid = *xactptr;
 		ptr[truelength].status = (*flagsptr >> bshift) & MXACT_MEMBER_XACT_BITMASK;
@@ -1926,13 +1925,21 @@ static Buffer
 ZeroMultiXactOffsetPage(int pageno, bool writeXlog)
 {
 	Buffer		buffer;
+	Page		page;
+	XLogRecPtr	recptr;
 
-	buffer = ZeroNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
 
-	MarkBufferDirty(buffer);
+	buffer = ZeroNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
+	page = BufferGetPage(buffer);
+	PageInitNREL(page, BLCKSZ, 0);
 
+	recptr = 0;
+	
 	if (writeXlog)
-		WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_OFF_PAGE);
+		recptr = WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_OFF_PAGE);
+
+	PageSetHeaderDataNonRel(page, pageno, recptr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION);
+	MarkBufferDirty(buffer);
 
 	return buffer;
 }
@@ -1944,14 +1951,20 @@ static Buffer
 ZeroMultiXactMemberPage(int pageno, bool writeXlog)
 {
 	Buffer		buffer;
+	Page 		page;
+	XLogRecPtr 	recptr;
 
 	buffer = ZeroNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno);
-
-	MarkBufferDirty(buffer);
-
+	page = BufferGetPage(buffer);
+	PageInitNREL(page, BLCKSZ, 0);
+		
+	recptr = 0;
 	if (writeXlog)
-		WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_MEM_PAGE);
-
+		recptr = WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_MEM_PAGE);
+	
+	PageSetHeaderDataNonRel(page, pageno, recptr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION);
+	MarkBufferDirty(buffer);
+	
 	return buffer;
 }
 
@@ -2041,10 +2054,10 @@ TrimMultiXact(void)
 		buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
 
 		LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
-		offptr = (MultiXactOffset *) BufferGetPage(buffer);
+		offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer)));
 		offptr += entryno;
 
-		MemSet(offptr, 0, BLCKSZ - (entryno * sizeof(MultiXactOffset)));
+		MemSet(offptr, 0, BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - (entryno * sizeof(MultiXactOffset)));
 
 		MarkBufferDirty(buffer);
 		UnlockReleaseBuffer(buffer);
@@ -2067,9 +2080,9 @@ TrimMultiXact(void)
 		buffer = ReadNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno);
 
 		LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
-		xidptr = (TransactionId *) (BufferGetPage(buffer) + memberoff);
+		xidptr = (TransactionId *) (PageGetContents(BufferGetPage(buffer)) + memberoff);
 
-		MemSet(xidptr, 0, BLCKSZ - memberoff);
+		MemSet(xidptr, 0, BLCKSZ - memberoff - MAXALIGN(SizeOfPageHeaderData));
 
 		/*
 		 * Note: we don't need to zero out the flag bits in the remaining
@@ -2419,7 +2432,6 @@ ExtendMultiXactMember(MultiXactOffset offset, int nmembers, Buffer ** buffers)
 		{
 			/* do a read buffer call to allocate space beforehand */
 			buf = ReadNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno);
-
 		}
 
 		if (buffers)
@@ -2718,7 +2730,7 @@ find_multixact_start(MultiXactId multi, MultiXactOffset *result)
 	buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno);
 
 	LockBuffer(buffer, BUFFER_LOCK_SHARE);
-	offptr = (MultiXactOffset *) BufferGetPage(buffer);
+	offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer)));
 	offptr += entryno;
 	offset = *offptr;
 	UnlockReleaseBuffer(buffer);
@@ -3138,12 +3150,16 @@ MultiXactOffsetPrecedes(MultiXactOffset offset1, MultiXactOffset offset2)
  * Write an xlog record reflecting the zeroing of either a MEMBERs or
  * OFFSETs page (info shows which)
  */
-static void
+static XLogRecPtr 
 WriteMZeroPageXlogRec(int pageno, uint8 info)
 {
+	XLogRecPtr recptr;
+	
 	XLogBeginInsert();
 	XLogRegisterData((char *) (&pageno), sizeof(int));
-	(void) XLogInsert(RM_MULTIXACT_ID, info);
+	recptr = XLogInsert(RM_MULTIXACT_ID, info);
+
+	return recptr;
 }
 
 /*
diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c
index 635e5368cd6..975fc47814e 100644
--- a/src/backend/access/transam/subtrans.c
+++ b/src/backend/access/transam/subtrans.c
@@ -51,7 +51,7 @@
  */
 
 /* We need four bytes per xact */
-#define SUBTRANS_XACTS_PER_PAGE (BLCKSZ / sizeof(TransactionId))
+#define SUBTRANS_XACTS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData) / sizeof(TransactionId))
 
 #define TransactionIdToPage(xid) ((xid) / (TransactionId) SUBTRANS_XACTS_PER_PAGE)
 #define TransactionIdToEntry(xid) ((xid) % (TransactionId) SUBTRANS_XACTS_PER_PAGE)
@@ -77,8 +77,9 @@ SubTransSetParent(TransactionId xid, TransactionId parent)
 
 	buffer = ReadNrelBuffer(NREL_SUBTRANS_REL_ID, pageno);
 
+
 	LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
-	ptr = (TransactionId *) BufferGetPage(buffer);
+	ptr = (TransactionId *) PageGetContents(BufferGetPage(buffer));
 	ptr += entryno;
 
 	/*
@@ -118,7 +119,8 @@ SubTransGetParent(TransactionId xid)
 	buffer = ReadNrelBuffer(NREL_SUBTRANS_REL_ID, pageno);
 
 
-	ptr = (TransactionId *) BufferGetPage(buffer);
+
+	ptr = (TransactionId *) PageGetContents(BufferGetPage(buffer));
 	ptr += entryno;
 
 	parent = *ptr;
@@ -208,9 +210,13 @@ static Buffer
 ZeroSUBTRANSPage(int pageno)
 {
 	Buffer		buffer;
+	Page 		page;
 
 	buffer = ZeroNrelBuffer(NREL_SUBTRANS_REL_ID, pageno);
 
+	page = BufferGetPage(buffer);
+	PageInitNREL(page, BLCKSZ, 0);
+
 	MarkBufferDirty(buffer);
 
 	return buffer;
diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c
index 723cf8f6e1c..6023133c809 100644
--- a/src/backend/commands/async.c
+++ b/src/backend/commands/async.c
@@ -165,7 +165,7 @@
  * than that, so changes in that data structure won't affect user-visible
  * restrictions.
  */
-#define NOTIFY_PAYLOAD_MAX_LENGTH	(BLCKSZ - NAMEDATALEN - 128)
+#define NOTIFY_PAYLOAD_MAX_LENGTH	(BLCKSZ - NAMEDATALEN - SizeOfPageHeaderData - 128)
 
 /*
  * Struct representing an entry in the global notify queue
@@ -215,7 +215,7 @@ typedef struct QueuePosition
 	((x).page == (y).page && (x).offset == (y).offset)
 
 #define QUEUE_POS_IS_ZERO(x) \
-	((x).page == 0 && (x).offset == 0)
+	((x).page == 0 && (x).offset == MAXALIGN(SizeOfPageHeaderData))
 
 /* choose logically smaller QueuePosition */
 #define QUEUE_POS_MIN(x,y) \
@@ -544,8 +544,8 @@ AsyncShmemInit(void)
 	if (!found)
 	{
 		/* First time through, so initialize it */
-		SET_QUEUE_POS(QUEUE_HEAD, 0, 0);
-		SET_QUEUE_POS(QUEUE_TAIL, 0, 0);
+		SET_QUEUE_POS(QUEUE_HEAD, 0, MAXALIGN(SizeOfPageHeaderData));
+		SET_QUEUE_POS(QUEUE_TAIL, 0, MAXALIGN(SizeOfPageHeaderData));
 		QUEUE_STOP_PAGE = 0;
 		QUEUE_FIRST_LISTENER = InvalidBackendId;
 		asyncQueueControl->lastQueueFillWarn = 0;
@@ -555,7 +555,7 @@ AsyncShmemInit(void)
 			QUEUE_BACKEND_PID(i) = InvalidPid;
 			QUEUE_BACKEND_DBOID(i) = InvalidOid;
 			QUEUE_NEXT_LISTENER(i) = InvalidBackendId;
-			SET_QUEUE_POS(QUEUE_BACKEND_POS(i), 0, 0);
+			SET_QUEUE_POS(QUEUE_BACKEND_POS(i), 0, MAXALIGN(SizeOfPageHeaderData));
 		}
 	}
 
@@ -1332,19 +1332,19 @@ asyncQueueAdvance(volatile QueuePosition *position, int entryLength)
 	 * written or read.
 	 */
 	offset += entryLength;
-	Assert(offset <= QUEUE_PAGESIZE);
+	Assert(offset <= QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData));
 
 	/*
 	 * In a second step check if another entry can possibly be written to the
 	 * page. If so, stay here, we have reached the next position. If not, then
 	 * we need to move on to the next page.
 	 */
-	if (offset + QUEUEALIGN(AsyncQueueEntryEmptySize) > QUEUE_PAGESIZE)
+	if (offset + QUEUEALIGN(AsyncQueueEntryEmptySize) > QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData))
 	{
 		pageno++;
 		if (pageno > QUEUE_MAX_PAGE)
 			pageno = 0;			/* wrap around */
-		offset = 0;
+		offset = MAXALIGN(SizeOfPageHeaderData); /* start at SizeOfPageHeaderData */
 		pageJump = true;
 	}
 
@@ -1399,6 +1399,7 @@ asyncQueueAddEntries(ListCell *nextNotify)
 	int			pageno;
 	int			offset;
 	Buffer		buffer;
+	Page		page;
 
 	/*
 	 * We work with a local copy of QUEUE_HEAD, which we write back to shared
@@ -1424,15 +1425,22 @@ asyncQueueAddEntries(ListCell *nextNotify)
 	pageno = QUEUE_POS_PAGE(queue_head);
 	if (QUEUE_POS_IS_ZERO(queue_head))
 	{
+
 		buffer = ZeroNrelBuffer(NREL_NOTIFY_REL_ID, pageno);
+		page = BufferGetPage(buffer);
+		PageInitNREL(page, BLCKSZ, 0);
+		PageSetHeaderDataNonRel(page, pageno, InvalidXLogRecPtr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION);
+
 	}
 	else
 	{
 		buffer = ReadNrelBuffer(NREL_NOTIFY_REL_ID, pageno);
 
+
 		LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 	}
 
+	
 	/* Note we mark the page dirty before writing in it */
 	MarkBufferDirty(buffer);
 
@@ -1446,7 +1454,7 @@ asyncQueueAddEntries(ListCell *nextNotify)
 		offset = QUEUE_POS_OFFSET(queue_head);
 
 		/* Check whether the entry really fits on the current page */
-		if (offset + qe.length <= QUEUE_PAGESIZE)
+		if (offset + qe.length <= QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData))
 		{
 			/* OK, so advance nextNotify past this item */
 			nextNotify = lnext(pendingNotifies->events, nextNotify);
@@ -1458,17 +1466,18 @@ asyncQueueAddEntries(ListCell *nextNotify)
 			 * only check dboid and since it won't match any reader's database
 			 * OID, they will ignore this entry and move on.
 			 */
-			qe.length = QUEUE_PAGESIZE - offset;
+			qe.length = QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData) - offset;
 			qe.dboid = InvalidOid;
 			qe.data[0] = '\0';	/* empty channel */
 			qe.data[1] = '\0';	/* empty payload */
 		}
 
 		/* Now copy qe into the shared buffer page */
-		memcpy(BufferGetPage(buffer) + offset,
+		memcpy(PageGetContents(BufferGetPage(buffer)) + offset,
 			   &qe,
 			   qe.length);
 
+
 		/* Advance queue_head appropriately, and detect if page is full */
 		if (asyncQueueAdvance(&(queue_head), qe.length))
 		{
@@ -1985,6 +1994,7 @@ asyncQueueReadAllNotifications(void)
 			 */
 			buffer = ReadNrelBuffer(NREL_NOTIFY_REL_ID, curpage);
 
+
 			if (curpage == QUEUE_POS_PAGE(head))
 			{
 				/* we only want to read as far as head */
@@ -1995,10 +2005,10 @@ asyncQueueReadAllNotifications(void)
 			else
 			{
 				/* fetch all the rest of the page */
-				copysize = QUEUE_PAGESIZE - curoffset;
+				copysize = QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData) - curoffset;
 			}
-			memcpy(page_buffer.buf + curoffset,
-				   BufferGetPage(buffer) + curoffset,
+			memcpy(PageGetContents(page_buffer.buf) +  curoffset,
+				   PageGetContents(BufferGetPage(buffer)) + curoffset,
 				   copysize);
 			ReleaseBuffer(buffer);
 
@@ -2068,7 +2078,7 @@ asyncQueueProcessPageEntries(volatile QueuePosition *current,
 		if (QUEUE_POS_EQUAL(thisentry, stop))
 			break;
 
-		qe = (AsyncQueueEntry *) (page_buffer + QUEUE_POS_OFFSET(thisentry));
+		qe = (AsyncQueueEntry *) (PageGetContents(page_buffer) + QUEUE_POS_OFFSET(thisentry));
 
 		/*
 		 * Advance *current over this message, possibly to the next page. As
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index d17910a8fe6..44ddce444d0 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -63,16 +63,12 @@
  * meantime we need some way to identify buffers that hold raw data (no
  * invasive LSN, no checksums).
  */
-#define BufferHasStandardPage(bufHdr)			\
-	((bufHdr)->tag.dbOid != 9)
-
-#define BufferHasExternalLSN(bufHdr)			\
-	!BufferHasStandardPage(bufHdr)
 
 /* Note: these two macros only work on shared buffers, not local ones! */
 #define BufHdrGetBlock(bufHdr)	((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ))
+
 #define BufferGetLSN(bufHdr) \
-	(BufferHasExternalLSN(bufHdr) ? BufferGetExternalLSN(bufHdr) : PageGetLSN(BufHdrGetBlock(bufHdr)))
+		PageGetLSN(BufHdrGetBlock(bufHdr))
 
 /* Note: this macro only works on local buffers, not shared ones! */
 #define LocalBufHdrGetBlock(bufHdr) \
@@ -1155,8 +1151,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 								IOOP_READ, io_start, 1);
 
 		/* check for garbage data */
-		if (BufferHasStandardPage(bufHdr) &&
-			(!PageIsVerifiedExtended((Page) bufBlock, blockNum,
+		if ((!(mode == RBM_TRIM)) && (!PageIsVerifiedExtended((Page) bufBlock, blockNum,
 									PIV_LOG_WARNING | PIV_REPORT_STAT)))
 		{
 			if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages)
@@ -1388,8 +1383,6 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 		return existing_buf_hdr;
 	}
 
-	if (BufferHasExternalLSN(victim_buf_hdr))
-		BufferSetExternalLSN(victim_buf_hdr, InvalidXLogRecPtr);
 
 	/*
 	 * Need to lock the buffer header too in order to change its tag.
@@ -3457,10 +3450,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
 	 * buffer, other processes might be updating hint bits in it, so we must
 	 * copy the page to private storage if we do checksumming.
 	 */
-	if (BufferHasStandardPage(buf))
-		bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
-	else
-		bufToWrite = bufBlock;
+	bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
 
 	io_start = pgstat_prepare_io_time();
 
@@ -3601,10 +3591,8 @@ BufferGetLSNAtomic(Buffer buffer)
 	Assert(BufferIsPinned(buffer));
 
 	buf_state = LockBufHdr(bufHdr);
-	if (BufferHasStandardPage(bufHdr))
-		lsn = PageGetLSN(page);
-	else
-		lsn = BufferGetExternalLSN(bufHdr);
+	lsn = PageGetLSN(page);
+	
 	UnlockBufHdr(bufHdr, buf_state);
 
 	return lsn;
@@ -4199,8 +4187,7 @@ FlushRelationBuffers(Relation rel)
 				errcallback.previous = error_context_stack;
 				error_context_stack = &errcallback;
 
-				if (BufferHasStandardPage(bufHdr))
-					PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
+				PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
 
 				io_start = pgstat_prepare_io_time();
 
diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c
index 2177386ca50..d1591dd9710 100644
--- a/src/backend/storage/lmgr/predicate.c
+++ b/src/backend/storage/lmgr/predicate.c
@@ -316,7 +316,7 @@
 
 #define SERIAL_PAGESIZE			BLCKSZ
 #define SERIAL_ENTRYSIZE			sizeof(SerCommitSeqNo)
-#define SERIAL_ENTRIESPERPAGE	(SERIAL_PAGESIZE / SERIAL_ENTRYSIZE)
+#define SERIAL_ENTRIESPERPAGE	((SERIAL_PAGESIZE - SizeOfPageHeaderData) / SERIAL_ENTRYSIZE)
 
 /*
  * Set maximum pages based on the number needed to track all transactions.
@@ -326,11 +326,12 @@
 #define SerialNextPage(page) (((page) >= SERIAL_MAX_PAGE) ? 0 : (page) + 1)
 
 #define SerialValue(buffer, xid) (*((SerCommitSeqNo *) \
-	(BufferGetPage(buffer) + \
+	(PageGetContents(BufferGetPage(buffer)) + \
 	((((uint32) (xid)) % SERIAL_ENTRIESPERPAGE) * SERIAL_ENTRYSIZE))))
 
 #define SerialPage(xid)	(((uint32) (xid)) / SERIAL_ENTRIESPERPAGE)
 
+
 typedef struct SerialControlData
 {
 	int			headPage;		/* newest initialized page */
@@ -781,7 +782,7 @@ SerialPagePrecedesLogicallyUnitTests(void)
 	 */
 	headPage = oldestPage;
 	targetPage = newestPage;
-	Assert(SerialPagePrecedesLogically(headPage, targetPage - 1));
+	Assert(SerialPagePrecedesLogically(headPage, targetPage - 2));
 #if 0
 	Assert(SerialPagePrecedesLogically(headPage, targetPage));
 #endif
@@ -878,6 +879,7 @@ SerialAdd(TransactionId xid, SerCommitSeqNo minConflictCommitSeqNo)
 		{
 			buffer = ZeroNrelBuffer(NREL_SERIAL_REL_ID, firstZeroPage);
 
+			PageSetHeaderDataNonRel(BufferGetPage(buffer), firstZeroPage, InvalidXLogRecPtr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION);
 			MarkBufferDirty(buffer);
 			UnlockReleaseBuffer(buffer);
 			firstZeroPage = SerialNextPage(firstZeroPage);
@@ -888,6 +890,7 @@ SerialAdd(TransactionId xid, SerCommitSeqNo minConflictCommitSeqNo)
 	{
 		buffer = ReadNrelBuffer(NREL_SERIAL_REL_ID, targetPage);
 
+
 		LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 	}
 
@@ -933,6 +936,7 @@ SerialGetMinConflictCommitSeqNo(TransactionId xid)
 	 */
 	buffer = ReadNrelBuffer(NREL_SERIAL_REL_ID, SerialPage(xid));
 
+
 	val = SerialValue(buffer, xid);
 	ReleaseBuffer(buffer);
 	LWLockRelease(SerialSLRULock);
diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c
index 9a302ddc30e..bc2f5f97b00 100644
--- a/src/backend/storage/page/bufpage.c
+++ b/src/backend/storage/page/bufpage.c
@@ -59,6 +59,27 @@ PageInit(Page page, Size pageSize, Size specialSize)
 	/* p->pd_prune_xid = InvalidTransactionId;		done by above MemSet */
 }
 
+void
+PageInitNREL(Page page, Size pageSize, Size specialSize)
+{
+	PageHeader	p = (PageHeader) page;
+
+	specialSize = MAXALIGN(specialSize);
+
+	Assert(pageSize == BLCKSZ);
+	Assert(pageSize > specialSize + SizeOfPageHeaderData);
+
+	/* Make sure all fields of page are zero, as well as unused space */
+	MemSet(p, 0, pageSize);
+
+	p->pd_flags = 0;
+	p->pd_lower = SizeOfPageHeaderData;
+	p->pd_upper = pageSize - specialSize;
+	p->pd_special = pageSize - specialSize;
+	PageSetPageSizeAndVersion(page, pageSize, PG_METAPAGE_LAYOUT_VERSION);
+	/* p->pd_prune_xid = InvalidTransactionId;		done by above MemSet */
+}
+
 
 /*
  * PageIsVerifiedExtended
@@ -103,7 +124,7 @@ PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags)
 		if (DataChecksumsEnabled())
 		{
 			checksum = pg_checksum_page((char *) page, blkno);
-
+			
 			if (checksum != p->pd_checksum)
 				checksum_failure = true;
 		}
diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c
index d1736028826..48dfb8505cc 100644
--- a/src/bin/pg_upgrade/file.c
+++ b/src/bin/pg_upgrade/file.c
@@ -20,6 +20,7 @@
 #endif
 
 #include "access/visibilitymapdefs.h"
+#include "access/nreldefs.h"
 #include "common/file_perm.h"
 #include "pg_upgrade.h"
 #include "storage/bufpage.h"
@@ -139,7 +140,6 @@ copyFile(const char *src, const char *dst,
 #endif							/* WIN32 */
 }
 
-
 /*
  * linkFile()
  *
@@ -316,6 +316,179 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile,
 	close(src_fd);
 }
 
+
+//create new file and initialize to 256K
+static int
+init_new_slru_file(FILE * fp)
+{ 
+	int fd;
+	int res;
+	
+	fd = fileno(fp);
+	res = ftruncate(fd, BLCKSZ * SLRU_PAGES_PER_SEGMENT);
+	return res;
+}
+
+int 
+copy_to_new_format(const char *old_subdir, const char *new_subdir, int element_size) 
+{
+	DIR *dr;
+	struct dirent *de;
+	
+	size_t 	read_items;
+	size_t 	n_items;
+	size_t 	to_read_items;
+	size_t 	write_items;
+	size_t 	pages_written;
+	size_t 	write_offset;
+
+	char * 	read_file;	
+	char * 	write_file;
+	
+	int 	res;
+	int 	write_file_segno;
+	int 	total_read_files;
+	
+	FILE * 	old_fd; 
+	FILE * 	new_fp;
+
+#define MAXBUFSIZE (BLCKSZ - SizeOfPageHeaderData)
+
+	char	old_path[MAXPGPATH];
+	char	new_path[MAXPGPATH];
+	PGAlignedBlock pg_buf;
+	BlockNumber blkno; //track aboslute block number
+	
+
+	struct 	dirent** all_dirents;
+
+
+	snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, old_subdir);
+	snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, new_subdir);
+
+	/* 
+	 * calculate the number of items that can fit 
+	 * inside BLCKSZ - SizeOfPageHeaderData
+	 */
+
+	n_items = (size_t) (MAXBUFSIZE / element_size); 
+	
+	/* number of items to read in at a time */
+	to_read_items = n_items;
+	
+	//bufsize = (size_t) (n_items * element_size); 
+	write_file_segno = 0; /* track which segment we are writing into */
+	write_offset = (size_t) (SizeOfPageHeaderData); 
+
+	dr = opendir((char *)old_path);
+	if (dr == NULL)
+		return -1;
+
+	write_file = psprintf("%s/%04X", new_path, write_file_segno);
+	
+	new_fp = fopen(write_file, "wb");
+	
+	if (new_fp == NULL)
+		return -1;
+
+	res = init_new_slru_file(new_fp);
+	if (res < 0)	
+		return -1;
+
+	if (res < 0)
+		return -1;
+
+	pages_written = 0;
+	blkno = 0; /* absolute blkno */
+
+	all_dirents = get_sorted_hex_files(old_path, &total_read_files);
+
+	write_items = 0;
+	for (int i = 0; i < total_read_files; i++)
+	{
+		de = all_dirents[i];
+		
+		if ((!strcmp((char *) de->d_name, ".")) || !strcmp((char *) de->d_name, ".."))
+	 	{
+			continue;
+	 	}
+
+		read_file = psprintf("%s/%s", (char *) old_path, (char *) de->d_name);
+		
+		old_fd = fopen(read_file, "rb");
+		read_items = 0;
+		
+		do 
+		{
+			memset(pg_buf.data, 0, BLCKSZ);		
+			read_items = fread(pg_buf.data + write_offset, element_size, to_read_items, old_fd);  /* how many items we read */
+
+			if (to_read_items == n_items) /* not finishing up a partial read */
+			{
+
+				((PageHeader) pg_buf.data)->pd_lower = SizeOfPageHeaderData;
+				((PageHeader) pg_buf.data)->pd_upper = BLCKSZ;
+				((PageHeader) pg_buf.data)->pd_special = BLCKSZ;
+				
+				((PageHeader) pg_buf.data)->pd_checksum =
+					pg_checksum_page(pg_buf.data, blkno);
+
+				fwrite(pg_buf.data, SizeOfPageHeaderData, 1, new_fp); /* write page header data */
+			}
+
+			
+			write_items += fwrite(pg_buf.data + write_offset, element_size, read_items, new_fp);  /* increment how many items written */
+						
+			if (write_items % n_items == 0 && write_items > 0 && errno == 0) /* finished writing into new page. */
+			{
+				if (write_items == n_items * SLRU_PAGES_PER_SEGMENT) /* end of segment */
+				{
+					fclose(new_fp);
+					pages_written = 0;
+					write_file_segno ++;
+					write_items = 0;
+					write_file = psprintf("%s/%04X", new_path, write_file_segno);
+					new_fp = fopen((char *)write_file, "wb");
+					res = init_new_slru_file(new_fp);
+					if (res < 0 || new_fp == NULL)
+					{
+						fclose(new_fp);
+						cleanup_dirents(all_dirents, total_read_files);
+						
+					}
+				} else { /* end of page but not segment */
+					//fflush(new_fp);
+					pages_written ++; 
+				}
+
+				blkno++; 
+
+				to_read_items = n_items; /* to_read_items = n */
+				res = fseek(new_fp, (pages_written * BLCKSZ), SEEK_SET);
+				if (res < 0)
+				{
+					fclose(new_fp);
+					cleanup_dirents(all_dirents, total_read_files);
+					return -1;
+				}
+			} else {
+				/*end of segment in source dir*/
+				if (read_items < to_read_items) 
+				{
+					to_read_items = n_items - read_items; /* read remaining items */
+				}
+			}
+		} while (!feof(old_fd));  /* until end of file */
+		fclose(old_fd);
+	}
+	fflush(new_fp);
+	
+	/* free memory malloc'd by scandir while sorting */
+	cleanup_dirents(all_dirents, total_read_files);
+
+	return 0;
+}
+
 void
 check_file_clone(void)
 {
diff --git a/src/bin/pg_upgrade/function.c b/src/bin/pg_upgrade/function.c
index dc8800c7cde..10c7fcafbd5 100644
--- a/src/bin/pg_upgrade/function.c
+++ b/src/bin/pg_upgrade/function.c
@@ -7,6 +7,7 @@
  *	src/bin/pg_upgrade/function.c
  */
 
+
 #include "postgres_fe.h"
 
 #include "access/transam.h"
@@ -42,6 +43,43 @@ library_name_compare(const void *p1, const void *p2)
 			((const LibraryInfo *) p2)->dbnum;
 }
 
+/*
+ * qsort comparator for hex filenames
+ */
+static int
+file_name_compare(const struct dirent ** de_1, const struct dirent ** de_2)
+{
+	int n1;
+	int n2;
+
+
+	char * fname_1;
+	char * fname_2;
+
+	fname_1 = (char *) (*de_1)->d_name;
+	fname_2 = (char *) (*de_2)->d_name;
+	
+	if ((strcmp(fname_1, ".")) || strcmp(fname_1, "..")) 
+	{
+		n1 = (int) strtol(fname_1, NULL, 16);
+	} else {
+		n1 = -1;
+	}
+
+	if ((strcmp(fname_2, ".")) || strcmp(fname_2, ".."))
+	{
+		n2 = (int) strtol(fname_2, NULL, 16);
+	} else {
+		n2 = -1;
+	}
+
+	if (n1 == n2) 
+	{
+		return 1; //arbitrarily select the first input
+	} else {
+		return n1 - n2;
+	}
+}
 
 /*
  * get_loadable_libraries()
@@ -109,6 +147,34 @@ get_loadable_libraries(void)
 }
 
 
+/*
+ * get_sorted_hex_files()
+ * given the filepath of a directory,
+ * return array of child dirents with hex filenames e.g '000A'
+ * in sorted order
+ */
+struct dirent** 
+get_sorted_hex_files(char * dr, int * size)
+{
+	struct dirent **entry_list;
+
+	*size = scandir(dr, &entry_list, NULL, file_name_compare);
+	if (*size < 0)
+	{
+		return NULL; //error
+	}
+	return entry_list;
+}
+
+void
+cleanup_dirents(struct dirent ** all_dirents, int total_read_files)
+{	
+	for (int i = 0; i < total_read_files; i++)
+	{
+		 free((struct dirent **) all_dirents[i]);
+	}
+	free((struct dirent **) all_dirents);
+}
 /*
  * check_loadable_libraries()
  *
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index 4562dafcff5..4925e4505ac 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -38,18 +38,22 @@
 
 #include "postgres_fe.h"
 
+#include <dirent.h>
 #include <time.h>
+#include <unistd.h>
 
 #ifdef HAVE_LANGINFO_H
 #include <langinfo.h>
 #endif
 
+#include "access/nreldefs.h"
 #include "catalog/pg_class_d.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/restricted_token.h"
 #include "fe_utils/string_utils.h"
 #include "pg_upgrade.h"
+#include "storage/bufpage.h"
 
 static void set_locale_and_encoding(void);
 static void prepare_new_cluster(void);
@@ -60,6 +64,10 @@ static void set_frozenxids(bool minmxid_only);
 static void make_outputdirs(char *pgdata);
 static void setup(char *argv0, bool *live_check);
 
+
+#define MAXBUFSIZE (BLCKSZ - SizeOfPageHeaderData)
+#define SLRU_PAGES_PER_SEGMENT 32
+
 ClusterInfo old_cluster,
 			new_cluster;
 OSInfo		os_info;
@@ -646,11 +654,36 @@ copy_xact_xlog_xid(void)
 	 * Copy old commit logs to new data dir. pg_clog has been renamed to
 	 * pg_xact in post-10 clusters.
 	 */
-	copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ?
-					  "pg_clog" : "pg_xact",
-					  GET_MAJOR_VERSION(new_cluster.major_version) <= 906 ?
-					  "pg_clog" : "pg_xact");
+	 
+	if (old_cluster.controldata.cat_ver <= CLOG_FORMATCHANGE_CAT_VER &&
+		new_cluster.controldata.cat_ver >= CLOG_FORMATCHANGE_CAT_VER)
+	{
+			int ret;
+			ret = copy_to_new_format(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ?
+			"pg_clog" : "pg_xact", 
+			"pg_xact", 1);
+			
+			if (ret < 0)
+			{
+				pg_fatal("could not reformat clog files");
+			}
+	}
 
+	if (old_cluster.controldata.cat_ver >= CLOG_FORMATCHANGE_CAT_VER &&
+		new_cluster.controldata.cat_ver >= CLOG_FORMATCHANGE_CAT_VER)
+	{
+		copy_subdir_files("pg_xact", "pg_xact");
+	}
+
+	if (old_cluster.controldata.cat_ver <= CLOG_FORMATCHANGE_CAT_VER &&
+		new_cluster.controldata.cat_ver <= CLOG_FORMATCHANGE_CAT_VER)
+	{
+		copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ?
+					"pg_clog" : "pg_xact",
+					GET_MAJOR_VERSION(new_cluster.major_version) <= 906 ?
+					"pg_clog" : "pg_xact");
+	}
+	
 	prep_status("Setting oldest XID for new cluster");
 	exec_prog(UTILITY_LOG_FILE, NULL, true, true,
 			  "\"%s/pg_resetwal\" -f -u %u \"%s\"",
@@ -706,23 +739,44 @@ copy_xact_xlog_xid(void)
 	}
 	else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
 	{
+		copy_to_new_format("pg_multixact/offsets", "pg_multixact/offsets", MULTIXACT_MEMBER_ENTRY_SIZE);
+		copy_to_new_format("pg_multixact/members", "pg_multixact/members", MULTIXACT_OFFSET_ENTRY_SIZE);	 
+
+
+		prep_status("Setting next multixact ID and offset for new cluster");
+
 		/*
+		 * we preserve all files and contents, so we must preserve both "next"
+		 * counters here and the oldest multi present on system.
+		 */
+		exec_prog(UTILITY_LOG_FILE, NULL, true, true,
+				  "\"%s/pg_resetwal\" -O %u -m %u,%u \"%s\"",
+				  new_cluster.bindir,
+				  old_cluster.controldata.chkpnt_nxtmxoff,
+				  old_cluster.controldata.chkpnt_nxtmulti,
+				  old_cluster.controldata.chkpnt_oldstMulti,
+				  new_cluster.pgdata);
+		check_ok();
+
+	}
+	/*else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
+	{
+		*
 		 * Remove offsets/0000 file created by initdb that no longer matches
 		 * the new multi-xid value.  "members" starts at zero so no need to
 		 * remove it.
-		 */
+		 *
 		remove_new_subdir("pg_multixact/offsets", false);
 
 		prep_status("Setting oldest multixact ID in new cluster");
-
-		/*
+ 		*
 		 * We don't preserve files in this case, but it's important that the
 		 * oldest multi is set to the latest value used by the old system, so
 		 * that multixact.c returns the empty set for multis that might be
 		 * present on disk.  We set next multi to the value following that; it
 		 * might end up wrapped around (i.e. 0) if the old cluster had
 		 * next=MaxMultiXactId, but multixact.c can cope with that just fine.
-		 */
+		 *
 		exec_prog(UTILITY_LOG_FILE, NULL, true, true,
 				  "\"%s/pg_resetwal\" -m %u,%u \"%s\"",
 				  new_cluster.bindir,
@@ -730,7 +784,8 @@ copy_xact_xlog_xid(void)
 				  old_cluster.controldata.chkpnt_nxtmulti,
 				  new_cluster.pgdata);
 		check_ok();
-	}
+		
+	}*/
 
 	/* now reset the wal archives in the new cluster */
 	prep_status("Resetting WAL archives");
@@ -742,7 +797,6 @@ copy_xact_xlog_xid(void)
 	check_ok();
 }
 
-
 /*
  *	set_frozenxids()
  *
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 3eea0139c74..7995d1e9635 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -7,6 +7,7 @@
 
 #include <unistd.h>
 #include <assert.h>
+#include <dirent.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 
@@ -113,7 +114,16 @@ extern char *output_files[];
  * version to this value.  pg_upgrade behavior depends on whether old and new
  * server versions are both newer than this, or only the new one is.
  */
-#define MULTIXACT_FORMATCHANGE_CAT_VER 201301231
+
+/*
+ * page header format change
+ */
+#define MULTIXACT_FORMATCHANGE_CAT_VER 202209141
+
+/* 
+ * page header format change 
+ */
+#define CLOG_FORMATCHANGE_CAT_VER 202210141
 
 /*
  * large object chunk size added to pg_controldata,
@@ -122,7 +132,7 @@ extern char *output_files[];
 #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942
 
 /*
- * change in JSONB format during 9.4 beta
+ * addition of page header 
  */
 #define JSONB_FORMAT_CHANGE_CAT_VER 201409291
 
@@ -386,6 +396,9 @@ void		rewriteVisibilityMap(const char *fromfile, const char *tofile,
 void		check_file_clone(void);
 void		check_hard_link(void);
 
+
+int 	copy_to_new_format(const char *old_subdir, const char *new_subdir, int element_size);
+
 /* fopen_priv() is no longer different from fopen() */
 #define fopen_priv(path, mode)	fopen(path, mode)
 
@@ -393,6 +406,8 @@ void		check_hard_link(void);
 
 void		get_loadable_libraries(void);
 void		check_loadable_libraries(void);
+void 		cleanup_dirents(struct dirent ** all_dirents, int total_read_files);
+struct dirent** 	get_sorted_hex_files(char * dr, int * size);
 
 /* info.c */
 
diff --git a/src/include/access/nreldefs.h b/src/include/access/nreldefs.h
new file mode 100644
index 00000000000..47bca8f48c3
--- /dev/null
+++ b/src/include/access/nreldefs.h
@@ -0,0 +1,19 @@
+/*-------------------------------------------------------------------------
+ *
+ * nreldefs.h
+ *		macros for accessing contents of "nrel" pages 
+ *
+ *
+ * Copyright (c) 2021-2022, PostgreSQL Global Development Group
+ *
+ * src/include/access/nreldefs.h
+ *
+ *--------------------------------------------------------------------------
+ */
+
+#define SLRU_PAGES_PER_SEGMENT 32
+
+#define MULTIXACT_MEMBER_ENTRY_SIZE 20 
+
+#define MULTIXACT_OFFSET_ENTRY_SIZE 8
+
diff --git a/src/include/access/slru.h b/src/include/access/slru.h
index 1a9ac3f1924..2cdaedb0f4f 100644
--- a/src/include/access/slru.h
+++ b/src/include/access/slru.h
@@ -15,6 +15,7 @@
 
 #include "access/xlogdefs.h"
 
+
 #include "storage/lwlock.h"
 #include "storage/sync.h"
 
@@ -184,4 +185,5 @@ extern bool SlruScanDirCbDeleteAll(SlruCtl ctl, char *filename, int segpage,
 								   void *data);
 
 
+
 #endif							/* SLRU_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index fb8561ae0e2..e5ed17633d2 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -47,8 +47,11 @@ typedef enum
 	RBM_ZERO_AND_CLEANUP_LOCK,	/* Like RBM_ZERO_AND_LOCK, but locks the page
 								 * in "cleanup" mode */
 	RBM_ZERO_ON_ERROR,			/* Read, but return an all-zeros page on error */
-	RBM_NORMAL_NO_LOG			/* Don't log page as invalid during WAL
+	RBM_NORMAL_NO_LOG,			/* Don't log page as invalid during WAL
 								 * replay; otherwise same as RBM_NORMAL */
+
+	RBM_TRIM				/*Read for TRIM functions in CLOG / MultiXact. 
+						  Don't validate checksum  or zero. */	
 } ReadBufferMode;
 
 /*
diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h
index 424ecba028f..a8cd8ce5168 100644
--- a/src/include/storage/bufpage.h
+++ b/src/include/storage/bufpage.h
@@ -201,6 +201,7 @@ typedef PageHeaderData *PageHeader;
  * handling pages.
  */
 #define PG_PAGE_LAYOUT_VERSION		4
+#define PG_METAPAGE_LAYOUT_VERSION  1
 #define PG_DATA_CHECKSUM_VERSION	1
 
 /* ----------------------------------------------------------------
@@ -302,6 +303,20 @@ PageSetPageSizeAndVersion(Page page, Size size, uint8 version)
 	((PageHeader) page)->pd_pagesize_version = size | version;
 }
 
+/*
+ * PageSetHeaderDataMinimal 
+ * Sets the LSN, page size and version, and checksum
+ */
+#define PageSetHeaderDataNonRel(page, pageno, lsn, size, version) \
+( \
+	PageSetLSN(page, lsn), \
+	PageSetPageSizeAndVersion(page, size, version), \
+	PageClearHasFreeLinePointers(page), \
+	PageSetChecksumInplace(page, pageno) \
+)
+
+
+
 /* ----------------
  *		page special data functions
  * ----------------
@@ -486,6 +501,8 @@ StaticAssertDecl(BLCKSZ == ((BLCKSZ / sizeof(size_t)) * sizeof(size_t)),
 				 "BLCKSZ has to be a multiple of sizeof(size_t)");
 
 extern void PageInit(Page page, Size pageSize, Size specialSize);
+extern void PageInitNREL(Page page, Size pageSize, Size specialSize);
+
 extern bool PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags);
 extern OffsetNumber PageAddItemExtended(Page page, Item item, Size size,
 										OffsetNumber offsetNumber, int flags);
-- 
2.40.1

