From 9656334d2dd6a85fde9c79736bd6615bb060e371 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Tue, 12 Apr 2022 17:54:25 +0900
Subject: [PATCH v2] Fix ABI/API break

Commit 57f618310f caused ABI break by changing the type and width of
PGXACT.delayChkpt and API break by changing the signature of some
public functions.  Restore the member and function signature to
previous state, and add delayChkptEnd to an existing gap in PGPROC
struct instead of PGXACT since there's no room for the new variable in
PGXACT.

Backpatch to 10, all supported branches.
---
 src/backend/access/transam/multixact.c  |  6 +--
 src/backend/access/transam/twophase.c   | 13 +++---
 src/backend/access/transam/xact.c       |  5 +-
 src/backend/access/transam/xlog.c       | 10 ++--
 src/backend/access/transam/xloginsert.c |  2 +-
 src/backend/catalog/storage.c           |  6 +--
 src/backend/storage/buffer/bufmgr.c     |  6 +--
 src/backend/storage/ipc/procarray.c     | 61 +++++++++++++++++++------
 src/backend/storage/lmgr/proc.c         |  6 ++-
 src/include/storage/proc.h              | 46 ++++---------------
 src/include/storage/procarray.h         |  8 ++--
 11 files changed, 87 insertions(+), 82 deletions(-)

diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c
index 1e52972bbf..cdaf499348 100644
--- a/src/backend/access/transam/multixact.c
+++ b/src/backend/access/transam/multixact.c
@@ -3069,8 +3069,8 @@ TruncateMultiXact(MultiXactId newOldestMulti, Oid newOldestMultiDB)
 	 * crash/basebackup, even though the state of the data directory would
 	 * require it.
 	 */
-	Assert((MyPgXact->delayChkpt & DELAY_CHKPT_START) == 0);
-	MyPgXact->delayChkpt |= DELAY_CHKPT_START;
+	Assert(!MyPgXact->delayChkpt);
+	MyPgXact->delayChkpt = true;
 
 	/* WAL log truncation */
 	WriteMTruncateXlogRec(newOldestMultiDB,
@@ -3096,7 +3096,7 @@ TruncateMultiXact(MultiXactId newOldestMulti, Oid newOldestMultiDB)
 	/* Then offsets */
 	PerformOffsetsTruncation(oldestMulti, newOldestMulti);
 
-	MyPgXact->delayChkpt &= ~DELAY_CHKPT_START;
+	MyPgXact->delayChkpt = false;
 
 	END_CRIT_SECTION();
 	LWLockRelease(MultiXactTruncationLock);
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index c61b2736a1..26a4faf388 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -478,7 +478,8 @@ MarkAsPreparingGuts(GlobalTransaction gxact, TransactionId xid, const char *gid,
 	}
 	pgxact->xid = xid;
 	pgxact->xmin = InvalidTransactionId;
-	pgxact->delayChkpt = 0;
+	pgxact->delayChkpt = false;
+	proc->delayChkptEnd = false;
 	pgxact->vacuumFlags = 0;
 	proc->pid = 0;
 	proc->databaseId = databaseid;
@@ -1159,8 +1160,7 @@ EndPrepare(GlobalTransaction gxact)
 
 	START_CRIT_SECTION();
 
-	Assert((MyPgXact->delayChkpt & DELAY_CHKPT_START) == 0);
-	MyPgXact->delayChkpt |= DELAY_CHKPT_START;
+	MyPgXact->delayChkpt = true;
 
 	XLogBeginInsert();
 	for (record = records.head; record != NULL; record = record->next)
@@ -1192,7 +1192,7 @@ EndPrepare(GlobalTransaction gxact)
 	 * checkpoint starting after this will certainly see the gxact as a
 	 * candidate for fsyncing.
 	 */
-	MyPgXact->delayChkpt &= ~DELAY_CHKPT_START;
+	MyPgXact->delayChkpt = false;
 
 	/*
 	 * Remember that we have this GlobalTransaction entry locked for us.  If
@@ -2285,8 +2285,7 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	START_CRIT_SECTION();
 
 	/* See notes in RecordTransactionCommit */
-	Assert((MyPgXact->delayChkpt & DELAY_CHKPT_START) == 0);
-	MyPgXact->delayChkpt |= DELAY_CHKPT_START;
+	MyPgXact->delayChkpt = true;
 
 	/*
 	 * Emit the XLOG commit record. Note that we mark 2PC commits as
@@ -2334,7 +2333,7 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	TransactionIdCommitTree(xid, nchildren, children);
 
 	/* Checkpoint can proceed now */
-	MyPgXact->delayChkpt &= ~DELAY_CHKPT_START;
+	MyPgXact->delayChkpt = false;
 
 	END_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index ccd99c38c2..25a3a4f97e 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -1247,9 +1247,8 @@ RecordTransactionCommit(void)
 		 * This makes checkpoint's determination of which xacts are delayChkpt
 		 * a bit fuzzy, but it doesn't matter.
 		 */
-		Assert((MyPgXact->delayChkpt & DELAY_CHKPT_START) == 0);
 		START_CRIT_SECTION();
-		MyPgXact->delayChkpt |= DELAY_CHKPT_START;
+		MyPgXact->delayChkpt = true;
 
 		SetCurrentTransactionStopTimestamp();
 
@@ -1350,7 +1349,7 @@ RecordTransactionCommit(void)
 	 */
 	if (markXidCommitted)
 	{
-		MyPgXact->delayChkpt &= ~DELAY_CHKPT_START;
+		MyPgXact->delayChkpt = false;
 		END_CRIT_SECTION();
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 51efc106a2..be6d9b239a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9022,27 +9022,25 @@ CreateCheckPoint(int flags)
 	 * and we will correctly flush the update below.  So we cannot miss any
 	 * xacts we need to wait for.
 	 */
-	vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_START);
+	vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
 	if (nvxids > 0)
 	{
 		do
 		{
 			pg_usleep(10000L);	/* wait for 10 msec */
-		} while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids,
-											  DELAY_CHKPT_START));
+		} while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids));
 	}
 	pfree(vxids);
 
 	CheckPointGuts(checkPoint.redo, flags);
 
-	vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_COMPLETE);
+	vxids = GetVirtualXIDsDelayingChkptEnd(&nvxids);
 	if (nvxids > 0)
 	{
 		do
 		{
 			pg_usleep(10000L);	/* wait for 10 msec */
-		} while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids,
-											  DELAY_CHKPT_COMPLETE));
+		} while (HaveVirtualXIDsDelayingChkptEnd(vxids, nvxids));
 	}
 	pfree(vxids);
 
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 6ff19814d4..579d8de775 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -899,7 +899,7 @@ XLogSaveBufferForHint(Buffer buffer, bool buffer_std)
 	/*
 	 * Ensure no checkpoint can change our view of RedoRecPtr.
 	 */
-	Assert((MyPgXact->delayChkpt & DELAY_CHKPT_START) != 0);
+	Assert(MyPgXact->delayChkpt);
 
 	/*
 	 * Update RedoRecPtr so that we can make the right decision
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 729fb92c5f..9497e1d88d 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -263,8 +263,8 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	 * the blocks to not exist on disk at all, but not for them to have the
 	 * wrong contents.
 	 */
-	Assert((MyPgXact->delayChkpt & DELAY_CHKPT_COMPLETE) == 0);
-	MyPgXact->delayChkpt |= DELAY_CHKPT_COMPLETE;
+	Assert(!MyProc->delayChkptEnd);
+	MyProc->delayChkptEnd = true;
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -312,7 +312,7 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
 
 	/* We've done all the critical work, so checkpoints are OK now. */
-	MyPgXact->delayChkpt &= ~DELAY_CHKPT_COMPLETE;
+	MyProc->delayChkptEnd = false;
 }
 
 /*
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 0b7bdb8634..bafe91ab0d 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -3469,9 +3469,7 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std)
 			 * essential that CreateCheckpoint waits for virtual transactions
 			 * rather than full transactionids.
 			 */
-			Assert((MyPgXact->delayChkpt & DELAY_CHKPT_START) == 0);
-			MyPgXact->delayChkpt |= DELAY_CHKPT_START;
-			delayChkpt = true;
+			MyPgXact->delayChkpt = delayChkpt = true;
 			lsn = XLogSaveBufferForHint(buffer, buffer_std);
 		}
 
@@ -3504,7 +3502,7 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std)
 		UnlockBufHdr(bufHdr, buf_state);
 
 		if (delayChkpt)
-			MyPgXact->delayChkpt &= ~DELAY_CHKPT_START;
+			MyPgXact->delayChkpt = false;
 
 		if (dirtied)
 		{
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 134b63f28b..9f5b26f201 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -171,6 +171,10 @@ static void KnownAssignedXidsReset(void);
 static inline void ProcArrayEndTransactionInternal(PGPROC *proc,
 								PGXACT *pgxact, TransactionId latestXid);
 static void ProcArrayGroupClearXid(PGPROC *proc, TransactionId latestXid);
+static VirtualTransactionId *GetVirtualXIDsDelayingChkptGuts(int *nvxids,
+															 bool start);
+static bool HaveVirtualXIDsDelayingChkptGuts(VirtualTransactionId *vxids,
+											 int nvxids, bool start);
 
 /*
  * Report shared-memory space needed by CreateSharedProcArray.
@@ -434,8 +438,9 @@ ProcArrayEndTransaction(PGPROC *proc, TransactionId latestXid)
 		/* must be cleared with xid/xmin: */
 		pgxact->vacuumFlags &= ~PROC_VACUUM_STATE_MASK;
 
-		/* be sure this is cleared in abort */
-		pgxact->delayChkpt = 0;
+		/* be sure these are cleared in abort */
+		pgxact->delayChkpt = false;
+		proc->delayChkptEnd = false;
 
 		proc->recoveryConflictPending = false;
 
@@ -459,8 +464,9 @@ ProcArrayEndTransactionInternal(PGPROC *proc, PGXACT *pgxact,
 	/* must be cleared with xid/xmin: */
 	pgxact->vacuumFlags &= ~PROC_VACUUM_STATE_MASK;
 
-	/* be sure this is cleared in abort */
-	pgxact->delayChkpt = 0;
+	/* be sure these are cleared in abort */
+	pgxact->delayChkpt = false;
+	proc->delayChkptEnd = false;
 
 	proc->recoveryConflictPending = false;
 
@@ -626,6 +632,7 @@ ProcArrayClearTransaction(PGPROC *proc)
 	/* redundant, but just in case */
 	pgxact->vacuumFlags &= ~PROC_VACUUM_STATE_MASK;
 	pgxact->delayChkpt = false;
+	proc->delayChkptEnd = false;
 
 	/* Clear the subtransaction-XID cache too */
 	pgxact->nxids = 0;
@@ -2265,8 +2272,10 @@ GetOldestSafeDecodingTransactionId(bool catalogOnly)
  * delaying checkpoint because they have critical actions in progress.
  *
  * Constructs an array of VXIDs of transactions that are currently in commit
- * critical sections, as shown by having specified delayChkpt bits set in their
- * PGXACT.
+ * critical sections, as shown by having the corresponding flag set in their
+ * PGPROC or PGXACT. GetVirtualXIDsDelayingChkpt checks PGXACT.delayChkpt, and
+ * GetVirtualXIDsDelayingChkptEnd checks PGPROC.delayChkptEnd. See PGPROC for
+ * the reason for the placement.
  *
  * Returns a palloc'd array that should be freed by the caller.
  * *nvxids is the number of valid entries.
@@ -2280,15 +2289,25 @@ GetOldestSafeDecodingTransactionId(bool catalogOnly)
  * for clearing of delayChkpt to propagate is unimportant for correctness.
  */
 VirtualTransactionId *
-GetVirtualXIDsDelayingChkpt(int *nvxids, int type)
+GetVirtualXIDsDelayingChkpt(int *nvxids)
+{
+	return GetVirtualXIDsDelayingChkptGuts(nvxids, true);
+}
+
+VirtualTransactionId *
+GetVirtualXIDsDelayingChkptEnd(int *nvxids)
+{
+	return GetVirtualXIDsDelayingChkptGuts(nvxids, false);
+}
+
+static VirtualTransactionId *
+GetVirtualXIDsDelayingChkptGuts(int *nvxids, bool start)
 {
 	VirtualTransactionId *vxids;
 	ProcArrayStruct *arrayP = procArray;
 	int			count = 0;
 	int			index;
 
-	Assert(type != 0);
-
 	/* allocate what's certainly enough result space */
 	vxids = (VirtualTransactionId *)
 		palloc(sizeof(VirtualTransactionId) * arrayP->maxProcs);
@@ -2301,7 +2320,7 @@ GetVirtualXIDsDelayingChkpt(int *nvxids, int type)
 		volatile PGPROC *proc = &allProcs[pgprocno];
 		volatile PGXACT *pgxact = &allPgXact[pgprocno];
 
-		if ((pgxact->delayChkpt & type) != 0)
+		if (start ? pgxact->delayChkpt : proc->delayChkptEnd)
 		{
 			VirtualTransactionId vxid;
 
@@ -2323,18 +2342,32 @@ GetVirtualXIDsDelayingChkpt(int *nvxids, int type)
  * This is used with the results of GetVirtualXIDsDelayingChkpt to see if any
  * of the specified VXIDs are still in critical sections of code.
  *
+ * HaveVirtualXIDsDelayingChkptEnd corresponds to
+ * GetVirtualXIDsDelayingChkptEnd above.
+ *
  * Note: this is O(N^2) in the number of vxacts that are/were delaying, but
  * those numbers should be small enough for it not to be a problem.
  */
 bool
-HaveVirtualXIDsDelayingChkpt(VirtualTransactionId *vxids, int nvxids, int type)
+HaveVirtualXIDsDelayingChkpt(VirtualTransactionId *vxids, int nvxids)
+{
+	return HaveVirtualXIDsDelayingChkptGuts(vxids, nvxids, true);
+}
+
+bool
+HaveVirtualXIDsDelayingChkptEnd(VirtualTransactionId *vxids, int nvxids)
+{
+	return HaveVirtualXIDsDelayingChkptGuts(vxids, nvxids, false);
+}
+
+static bool
+HaveVirtualXIDsDelayingChkptGuts(VirtualTransactionId *vxids, int nvxids,
+								 bool start)
 {
 	bool		result = false;
 	ProcArrayStruct *arrayP = procArray;
 	int			index;
 
-	Assert(type != 0);
-
 	LWLockAcquire(ProcArrayLock, LW_SHARED);
 
 	for (index = 0; index < arrayP->numProcs; index++)
@@ -2346,7 +2379,7 @@ HaveVirtualXIDsDelayingChkpt(VirtualTransactionId *vxids, int nvxids, int type)
 
 		GET_VXID_FROM_PGPROC(vxid, *proc);
 
-		if ((pgxact->delayChkpt & type) != 0 &&
+		if ((start ? pgxact->delayChkpt : proc->delayChkptEnd) &&
 			VirtualTransactionIdIsValid(vxid))
 		{
 			int			i;
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index e5370df019..0efee8411e 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -377,7 +377,8 @@ InitProcess(void)
 	MyProc->databaseId = InvalidOid;
 	MyProc->roleId = InvalidOid;
 	MyProc->isBackgroundWorker = IsBackgroundWorker;
-	MyPgXact->delayChkpt = 0;
+	MyPgXact->delayChkpt = false;
+	MyProc->delayChkptEnd = false;
 	MyPgXact->vacuumFlags = 0;
 	/* NB -- autovac launcher intentionally does not set IS_AUTOVACUUM */
 	if (IsAutoVacuumWorkerProcess())
@@ -550,7 +551,8 @@ InitAuxiliaryProcess(void)
 	MyProc->databaseId = InvalidOid;
 	MyProc->roleId = InvalidOid;
 	MyProc->isBackgroundWorker = IsBackgroundWorker;
-	MyPgXact->delayChkpt = 0;
+	MyPgXact->delayChkpt = false;
+	MyProc->delayChkptEnd = false;
 	MyPgXact->vacuumFlags = 0;
 	MyProc->lwWaiting = false;
 	MyProc->lwWaitMode = 0;
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index d8dd7bf5e1..e6d0f74458 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -75,41 +75,6 @@ struct XidCache
  */
 #define INVALID_PGPROCNO		PG_INT32_MAX
 
-/*
- * Flags for PGPROC.delayChkpt
- *
- * These flags can be used to delay the start or completion of a checkpoint
- * for short periods. A flag is in effect if the corresponding bit is set in
- * the PGPROC of any backend.
- *
- * For our purposes here, a checkpoint has three phases: (1) determine the
- * location to which the redo pointer will be moved, (2) write all the
- * data durably to disk, and (3) WAL-log the checkpoint.
- *
- * Setting DELAY_CHKPT_START prevents the system from moving from phase 1
- * to phase 2. This is useful when we are performing a WAL-logged modification
- * of data that will be flushed to disk in phase 2. By setting this flag
- * before writing WAL and clearing it after we've both written WAL and
- * performed the corresponding modification, we ensure that if the WAL record
- * is inserted prior to the new redo point, the corresponding data changes will
- * also be flushed to disk before the checkpoint can complete. (In the
- * extremely common case where the data being modified is in shared buffers
- * and we acquire an exclusive content lock on the relevant buffers before
- * writing WAL, this mechanism is not needed, because phase 2 will block
- * until we release the content lock and then flush the modified data to
- * disk.)
- *
- * Setting DELAY_CHKPT_COMPLETE prevents the system from moving from phase 2
- * to phase 3. This is useful if we are performing a WAL-logged operation that
- * might invalidate buffers, such as relation truncation. In this case, we need
- * to ensure that any buffers which were invalidated and thus not flushed by
- * the checkpoint are actaully destroyed on disk. Replay can cope with a file
- * or block that doesn't exist, but not with a block that has the wrong
- * contents.
- */
-#define DELAY_CHKPT_START		(1<<0)
-#define DELAY_CHKPT_COMPLETE	(1<<1)
-
 /*
  * Each backend has a PGPROC struct in shared memory.  There is also a list of
  * currently-unused PGPROC structs that will be reallocated to new backends.
@@ -181,6 +146,14 @@ struct PGPROC
 	 */
 	XLogRecPtr	waitLSN;		/* waiting for this LSN or higher */
 	int			syncRepState;	/* wait state for sync rep */
+
+	/*
+	 * This is a variable in duality with PGXACT.delayChkpt, but placed here to
+	 * prevent ABI breakage by moving other struct members of the two structs
+	 * around.
+	 */
+	bool		delayChkptEnd;	/* true if this proc delays checkpoint end */
+
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
@@ -252,7 +225,8 @@ typedef struct PGXACT
 
 	uint8		vacuumFlags;	/* vacuum-related flags, see above */
 	bool		overflowed;
-	int			delayChkpt;		/* for DELAY_CHKPT_* flags */
+	bool		delayChkpt;		/* true if this proc delays checkpoint start;
+								 * previously called InCommit */
 
 	uint8		nxids;
 } PGXACT;
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 2b60b27604..9a139c5f61 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -92,9 +92,11 @@ extern TransactionId GetOldestXmin(Relation rel, int flags);
 extern TransactionId GetOldestActiveTransactionId(void);
 extern TransactionId GetOldestSafeDecodingTransactionId(bool catalogOnly);
 
-extern VirtualTransactionId *GetVirtualXIDsDelayingChkpt(int *nvxids, int type);
-extern bool HaveVirtualXIDsDelayingChkpt(VirtualTransactionId *vxids,
-										 int nvxids, int type);
+extern VirtualTransactionId *GetVirtualXIDsDelayingChkpt(int *nvxids);
+extern bool HaveVirtualXIDsDelayingChkpt(VirtualTransactionId *vxids, int nvxids);
+extern VirtualTransactionId *GetVirtualXIDsDelayingChkptEnd(int *nvxids);
+extern bool HaveVirtualXIDsDelayingChkptEnd(VirtualTransactionId *vxids,
+											int nvxids);
 
 extern PGPROC *BackendPidGetProc(int pid);
 extern PGPROC *BackendPidGetProcWithLock(int pid);
-- 
2.27.0

