From 7afa91ece42a6ed2f57057ba912a8fd008c531f8 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 25 Feb 2026 13:41:42 +0500 Subject: [PATCH v2 2/3] walreceiver: add WALRCV_SWITCHING_TIMELINE state When the primary signals end-of-timeline, walreceiver calls walrcv_endstreaming() and then fetches the new timeline history file (WalRcvFetchTimeLineHistoryFiles). During this window the walreceiver state is still WALRCV_STREAMING, which causes a problem: startup may be sleeping in WAIT_EVENT_RECOVERY_WAL_STREAM waiting for data that will never arrive (walreceiver is fetching history, not delivering WAL). When startup eventually wakes and sees WalRcvStreaming() true while in the XLOG_FROM_STREAM handler, it kills walreceiver via XLogShutdownWalRcv(). The new walreceiver has to reconnect and re-request the timeline switch unnecessarily. Add WALRCV_SWITCHING_TIMELINE, a state walreceiver enters just before calling WalRcvFetchTimeLineHistoryFiles(). Since WalRcvStreaming() does not include this state, startup takes the "no kill" else branch (ResetInstallXLogFileSegmentActive) and backs off into wal_retrieve_retry_interval instead of killing walreceiver. Call WakeupRecovery() immediately after the state transition. Without this, startup remains stuck in WAIT_EVENT_RECOVERY_WAL_STREAM indefinitely: walreceiver only calls WakeupRecovery() from WalRcvWaitForStartPosition(), which runs after the history fetch. Guard RequestXLogStreaming() against WALRCV_SWITCHING_TIMELINE: because WALRCV_SWITCHING_TIMELINE is not considered "streaming", the XLOG_FROM_STREAM failure path no longer calls XLogShutdownWalRcv() to drain walreceiver before trying XLOG_FROM_ARCHIVE. When startup cycles back to XLOG_FROM_STREAM with startWalReceiver=true, walreceiver may still be in WALRCV_SWITCHING_TIMELINE, which would Assert-fail the STOPPED||WAITING check. Return early instead. --- src/backend/access/transam/xlogrecovery.c | 12 +++++--- src/backend/replication/walreceiver.c | 32 +++++++++++++++++++++- src/backend/replication/walreceiverfuncs.c | 14 ++++++++++ src/include/replication/walreceiver.h | 3 ++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index b4c4e87e0fb..3bfad82703f 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3715,10 +3715,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * WAL that we restore from archive. * * If walreceiver is actively streaming (or attempting to - * connect), we must shut it down. However, if it's - * already in WAITING state (e.g., due to timeline - * divergence), we only need to reset the install flag to - * allow archive restoration. + * connect), we must shut it down. However, if it's in + * WAITING state (e.g., due to timeline divergence) or in + * SWITCHING_TIMELINE state (fetching the timeline history + * file from the primary after end-of-timeline), we only + * need to reset the install flag to allow archive + * restoration; in the latter case walreceiver will soon + * transition to WAITING and wake us up with new + * instructions. */ if (WalRcvStreaming()) XLogShutdownWalRcv(); diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 10e64a7d1f4..4c674979ff3 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -207,6 +207,7 @@ WalReceiverMain(const void *startup_data, size_t startup_data_len) case WALRCV_CONNECTING: case WALRCV_WAITING: case WALRCV_STREAMING: + case WALRCV_SWITCHING_TIMELINE: case WALRCV_RESTARTING: default: /* Shouldn't happen */ @@ -609,7 +610,32 @@ WalReceiverMain(const void *startup_data, size_t startup_data_len) * If the server had switched to a new timeline that we didn't * know about when we began streaming, fetch its timeline history * file now. + * + * Transition to WALRCV_SWITCHING_TIMELINE while the fetch is in + * progress. This tells the startup process that we are still + * doing useful work related to the timeline switch and should not + * be shut down: from startup's perspective, + * WALRCV_SWITCHING_TIMELINE is not "streaming" (WalRcvStreaming() + * returns false), so it will not call XLogShutdownWalRcv() + * prematurely. Once the fetch completes we move to + * WALRCV_WAITING via WalRcvWaitForStartPosition() as usual. */ + if (startpointTLI != primaryTLI) + { + SpinLockAcquire(&walrcv->mutex); + Assert(walrcv->walRcvState == WALRCV_STREAMING); + walrcv->walRcvState = WALRCV_SWITCHING_TIMELINE; + SpinLockRelease(&walrcv->mutex); + + /* + * Notify the startup process that we have stopped streaming. + * Without this, startup would remain stuck in + * WAIT_EVENT_RECOVERY_WAL_STREAM waiting for new data that + * will never arrive while we are fetching history. + */ + WakeupRecovery(); + + } WalRcvFetchTimeLineHistoryFiles(startpointTLI, primaryTLI); } else @@ -661,7 +687,8 @@ WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI) SpinLockAcquire(&walrcv->mutex); state = walrcv->walRcvState; - if (state != WALRCV_STREAMING && state != WALRCV_CONNECTING) + if (state != WALRCV_STREAMING && state != WALRCV_CONNECTING && + state != WALRCV_SWITCHING_TIMELINE) { SpinLockRelease(&walrcv->mutex); if (state == WALRCV_STOPPING) @@ -804,6 +831,7 @@ WalRcvDie(int code, Datum arg) SpinLockAcquire(&walrcv->mutex); Assert(walrcv->walRcvState == WALRCV_STREAMING || walrcv->walRcvState == WALRCV_CONNECTING || + walrcv->walRcvState == WALRCV_SWITCHING_TIMELINE || walrcv->walRcvState == WALRCV_RESTARTING || walrcv->walRcvState == WALRCV_STARTING || walrcv->walRcvState == WALRCV_WAITING || @@ -1407,6 +1435,8 @@ WalRcvGetStateString(WalRcvState state) return "connecting"; case WALRCV_STREAMING: return "streaming"; + case WALRCV_SWITCHING_TIMELINE: + return "switching timeline"; case WALRCV_WAITING: return "waiting"; case WALRCV_RESTARTING: diff --git a/src/backend/replication/walreceiverfuncs.c b/src/backend/replication/walreceiverfuncs.c index 42e3e170bc0..54d47c7dcd3 100644 --- a/src/backend/replication/walreceiverfuncs.c +++ b/src/backend/replication/walreceiverfuncs.c @@ -213,6 +213,7 @@ ShutdownWalRcv(void) case WALRCV_CONNECTING: case WALRCV_STREAMING: + case WALRCV_SWITCHING_TIMELINE: case WALRCV_WAITING: case WALRCV_RESTARTING: walrcv->walRcvState = WALRCV_STOPPING; @@ -277,6 +278,19 @@ RequestXLogStreaming(TimeLineID tli, XLogRecPtr recptr, const char *conninfo, SpinLockAcquire(&walrcv->mutex); + /* + * If walreceiver is in WALRCV_SWITCHING_TIMELINE, it's fetching the + * timeline history file from the primary after detecting end-of-timeline. + * It will transition to WALRCV_WAITING on its own and then call + * WakeupRecovery(), at which point the startup process should call us + * again. Don't interrupt the fetch now. + */ + if (walrcv->walRcvState == WALRCV_SWITCHING_TIMELINE) + { + SpinLockRelease(&walrcv->mutex); + return; + } + /* It better be stopped if we try to restart it */ Assert(walrcv->walRcvState == WALRCV_STOPPED || walrcv->walRcvState == WALRCV_WAITING); diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 9b9bd916314..df30b35cd06 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -49,6 +49,9 @@ typedef enum * initialized yet */ WALRCV_CONNECTING, /* connecting to upstream server */ WALRCV_STREAMING, /* walreceiver is streaming */ + WALRCV_SWITCHING_TIMELINE, /* fetching timeline history from primary + * after end-of-timeline; not yet waiting for + * orders from startup */ WALRCV_WAITING, /* stopped streaming, waiting for orders */ WALRCV_RESTARTING, /* asked to restart streaming */ WALRCV_STOPPING, /* requested to stop, but still running */ -- 2.51.2