From 0a6b047292e08a4abe4a0eee4fa8a5ba6d68ead2 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Sep 2026 14:57:13 -0400 Subject: [PATCH] Fix async archive-push stalling on a zero-size backup history file. The archive-push-batch-size option converts the batch size to a segment count using the size of the first file in the ready list, but that list also contains backup and timeline history files. PostgreSQL does not sync backup history files, so a cluster copied from a storage snapshot may contain one with zero size, which caused every asynchronous run to error and prevented any WAL from being archived. Use the size of the first ready WAL segment instead and skip the batch size when no segment is ready, since the remaining files are small. A zero-size WAL segment is still an error. --- doc/xml/release/2020s/2026/2.60.0.xml | 13 +++++++ doc/xml/release/contributor.xml | 5 +++ src/command/archive/push/push.c | 36 +++++++++++++---- test/src/module/command/archive/pushTest.c | 45 ++++++++++++++++++++++ 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/doc/xml/release/2020s/2026/2.60.0.xml b/doc/xml/release/2020s/2026/2.60.0.xml index 2969ebd5f..41c13ec08 100644 --- a/doc/xml/release/2020s/2026/2.60.0.xml +++ b/doc/xml/release/2020s/2026/2.60.0.xml @@ -68,6 +68,19 @@

Disable read retry when filters are passed to the driver.

+ + + + + + + + + + + +

Fix async archive-push stalling on a zero-size backup history file.

+ diff --git a/doc/xml/release/contributor.xml b/doc/xml/release/contributor.xml index 04ec549de..ac599d353 100644 --- a/doc/xml/release/contributor.xml +++ b/doc/xml/release/contributor.xml @@ -95,6 +95,11 @@ AndrewJackson2020 + + Andrew Kroh + andrewkroh + + Andrew L'Ecuyer andrewlecuyer diff --git a/src/command/archive/push/push.c b/src/command/archive/push/push.c index 67f8a8805..dfcf66005 100644 --- a/src/command/archive/push/push.c +++ b/src/command/archive/push/push.c @@ -553,18 +553,38 @@ cmdArchivePushAsync(void) // If not dropping then limit the number of WAL files pushed in a single run to archive-push-batch-size. The queue is // only rechecked at the start of each run so this bounds how large it can grow before the next run rechecks it. Convert - // the size to a segment count using the size of the first ready segment and always push at least one segment. A ready - // WAL segment should never be missing or zero size, so error immediately if it is. + // the size to a segment count using the size of the first ready segment and always push at least one file. Files that + // are not segments also count against the batch, so a run may push only those files when the batch size is small, but + // they are acknowledged and drop out of the list for the next run. if (!drop) { - const uint64_t segmentSize = storageInfoP( - storagePg(), strNewFmt("%s/%s", strZ(jobData.walPath), strZ(strLstGet(jobData.walFileList, 0)))).size; - CHECK_FMT(FormatError, segmentSize != 0, "size of WAL segment '%s' is 0", strZ(strLstGet(jobData.walFileList, 0))); + // Find the first ready WAL segment. The ready list also contains backup and timeline history files, which are + // smaller than a segment and may even be zero size when PostgreSQL did not sync them before the cluster was + // copied, so they cannot be used to determine the segment size. + const String *walSegment = NULL; - const uint64_t batchMax = cfgOptionUInt64(cfgOptArchivePushBatchSize) / segmentSize; + for (unsigned int walFileIdx = 0; walFileIdx < strLstSize(jobData.walFileList); walFileIdx++) + { + if (walIsSegment(strLstGet(jobData.walFileList, walFileIdx))) + { + walSegment = strLstGet(jobData.walFileList, walFileIdx); + break; + } + } - while (strLstSize(jobData.walFileList) > 1 && strLstSize(jobData.walFileList) > batchMax) - strLstRemoveIdx(jobData.walFileList, strLstSize(jobData.walFileList) - 1); + // Apply the batch size when a segment is ready. When none is ready every file is small so there is nothing to + // limit. A ready WAL segment should never be missing or zero size, so error immediately if it is. + if (walSegment != NULL) + { + const uint64_t segmentSize = storageInfoP( + storagePg(), strNewFmt("%s/%s", strZ(jobData.walPath), strZ(walSegment))).size; + CHECK_FMT(FormatError, segmentSize != 0, "size of WAL segment '%s' is 0", strZ(walSegment)); + + const uint64_t batchMax = cfgOptionUInt64(cfgOptArchivePushBatchSize) / segmentSize; + + while (strLstSize(jobData.walFileList) > 1 && strLstSize(jobData.walFileList) > batchMax) + strLstRemoveIdx(jobData.walFileList, strLstSize(jobData.walFileList) - 1); + } } LOG_INFO_FMT( diff --git a/test/src/module/command/archive/pushTest.c b/test/src/module/command/archive/pushTest.c index 5dacac7bf..d3b1a5d16 100644 --- a/test/src/module/command/archive/pushTest.c +++ b/test/src/module/command/archive/pushTest.c @@ -1135,6 +1135,51 @@ testRun(void) TEST_ERROR( cmdArchivePushAsync(), FormatError, "size of WAL segment '000000010000000100000016' is 0"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("zero-size backup history file before the first ready segment"); + + HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true); + HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT); + HRN_STORAGE_PATH_REMOVE(storagePgWrite(), "pg_xlog/archive_status", .recurse = true); + HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_xlog/archive_status"); + + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/000000010000000100000018.00000028.backup"); + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000018.00000028.backup.ready"); + HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000019", walBufferBatch); + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000019.ready"); + + HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync); + + TEST_RESULT_VOID(cmdArchivePushAsync(), "push backup history file and WAL segment"); + TEST_RESULT_LOG( + "P00 INFO: push 2 WAL file(s) to archive: 000000010000000100000018.00000028.backup...000000010000000100000019\n" + "P01 DETAIL: pushed WAL file '000000010000000100000018.00000028.backup' to the archive\n" + "P01 DETAIL: pushed WAL file '000000010000000100000019' to the archive"); + + TEST_STORAGE_LIST( + storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, + "000000010000000100000018.00000028.backup.ok\n" + "000000010000000100000019.ok\n", + .comment = "check status files"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("no ready segments"); + + HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true); + HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT); + HRN_STORAGE_PATH_REMOVE(storagePgWrite(), "pg_xlog/archive_status", .recurse = true); + HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_xlog/archive_status"); + + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/00000002.history"); + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/00000002.history.ready"); + + TEST_RESULT_VOID(cmdArchivePushAsync(), "push history file"); + TEST_RESULT_LOG( + "P00 INFO: push 1 WAL file(s) to archive: 00000002.history\n" + "P01 DETAIL: pushed WAL file '00000002.history' to the archive"); + + TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, "00000002.history.ok\n", .comment = "check status files"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("archive-push-queue-max is checked against the full queue, not the batch");