database/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-10-23 21:28:56 +03:00
Code Activity

Use original size to determine if a file should be bundled.

Browse Source 
The bundling decision and the process queue ordering used ManifestFile.size, which manifestBuildIncr sets to the prior file size when block incremental values are preserved. A file that grew since the prior backup was measured by its prior size, so a file well above repo-bundle-limit could be stored in a bundle and sorted with the small files rather than ahead of them.

Use sizeOriginal instead since that is the size found during manifest build and therefore the number of bytes that will be copied. This matches the rest of backup processing, which already uses sizeOriginal for the copy limit, the bundle size accumulator, and progress.
This commit is contained in:
David Steele
2026-09-03 12:31:57 +03:00
committed by GitHub
parent 8687b12560
commit 340d58bb49
3 changed files with 45 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release/2020s/2026/2.60.0.xml
View File

@@ -63,6 +63,17 @@
<p>Reduce the manifest checks made when reconstructing <file>backup.info</file>.</p>
</release-item>
<release-item>
<github-pull-request id="2843"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="douglas.j.hunley"/>
</release-item-contributor-list>
<p>Use original size to determine if a file should be bundled.</p>
</release-item>
<release-item>
<github-pull-request id="2830"/>

13
src/command/backup/process.c.inc
View File

@@ -389,9 +389,10 @@ backupProcessQueueComparator(const void *const item1, const void *const item2)
const ManifestFile file1 = manifestFileUnpack(backupProcessQueueComparatorManifest, *(const ManifestFilePack *const *)item1);
const ManifestFile file2 = manifestFileUnpack(backupProcessQueueComparatorManifest, *(const ManifestFilePack *const *)item2);
// A file is bundled when bundling is enabled and it is within the bundle limit; larger files are stored unbundled
const bool file1Bundle = backupProcessQueueComparatorBundle && file1.size <= backupProcessQueueComparatorBundleLimit;
const bool file2Bundle = backupProcessQueueComparatorBundle && file2.size <= backupProcessQueueComparatorBundleLimit;
// A file is bundled when bundling is enabled and it is within the bundle limit; larger files are stored unbundled. Use the
// original size since that is how many bytes will be copied -- size may have been set to the prior size during manifest build.
const bool file1Bundle = backupProcessQueueComparatorBundle && file1.sizeOriginal <= backupProcessQueueComparatorBundleLimit;
const bool file2Bundle = backupProcessQueueComparatorBundle && file2.sizeOriginal <= backupProcessQueueComparatorBundleLimit;
// Order unbundled files (each copied as its own job) before bundled files
if (file1Bundle != file2Bundle)
@@ -401,9 +402,9 @@ backupProcessQueueComparator(const void *const item1, const void *const item2)
// backup where one process is still copying a large file after the others are idle. This only matters when process-max > 1.
if (!file1Bundle)
{
if (file1.size < file2.size)
if (file1.sizeOriginal < file2.sizeOriginal)
FUNCTION_TEST_RETURN(INT, 1);
else if (file1.size > file2.size)
else if (file1.sizeOriginal > file2.sizeOriginal)
FUNCTION_TEST_RETURN(INT, -1);
// If size is equal then use name to generate a deterministic ordering (names are unique)
@@ -657,7 +658,7 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
{
param = protocolPackNew();
if (bundle && file.size <= jobData->bundleLimit)
if (bundle && file.sizeOriginal <= jobData->bundleLimit)
{
pckWriteStrP(param, backupFileRepoPathP(jobData->backupLabel, .bundleId = jobData->bundleId));
pckWriteU64P(param, jobData->bundleId);

34
test/src/module/command/backupTest.c
View File

@@ -3698,6 +3698,13 @@ testRun(void)
HRN_STORAGE_PUT(storagePgWrite(), "grow-to-block-incr", file, .timeModified = backupTimeStart);
// Block incremental file within the bundle limit that will grow past it
file = bufNew(BLOCK_MIN_FILE_SIZE);
memset(bufPtr(file), 88, bufSize(file));
bufUsedSet(file, bufSize(file));
HRN_STORAGE_PUT(storagePgWrite(), "grow-past-bundle-limit", file, .timeModified = backupTimeStart);
// Run backup
hrnBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 2, .walSwitch = true);
TEST_RESULT_VOID(hrnCmdBackup(), "backup");
@@ -3722,17 +3729,18 @@ testRun(void)
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-same (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-below (bundle 1/16403, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-block (bundle 1/32809, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/49215, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/65629, 2B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/normal-same (bundle 1/65631, 4B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/65635, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/82018, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-past-bundle-limit (bundle 1/49215, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/65621, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/82035, 2B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/normal-same (bundle 1/82037, 4B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/82041, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/98424, 8KB, [PCT]) checksum [SHA1]\n"
"P00 INFO: execute backup stop and wait for all WAL segments to archive\n"
"P00 INFO: backup stop archive = 0000000105DC08C000000001, lsn = 5dc08c0/300000\n"
"P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n"
"P00 INFO: check archive for segment(s) 0000000105DC08C000000000:0000000105DC08C000000001\n"
"P00 INFO: new backup label = 20191103-165320F\n"
"P00 INFO: full backup size = [SIZE], file total = 12");
"P00 INFO: full backup size = [SIZE], file total = 13");
TEST_RESULT_STR_Z(
testBackupValidateP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
@@ -3743,6 +3751,7 @@ testRun(void)
"bundle/1/pg_data/block-incr-shrink-below {s=16384, m=0:{0,1}}\n"
"bundle/1/pg_data/block-incr-shrink-block {s=16384, m=0:{0,1}}\n"
"bundle/1/pg_data/global/pg_control {s=8192}\n"
"bundle/1/pg_data/grow-past-bundle-limit {s=16384, m=0:{0,1}}\n"
"bundle/1/pg_data/grow-to-block-incr {s=16383}\n"
"bundle/1/pg_data/normal-same {s=4}\n"
"pg_data/backup_label {s=17, ts=+2}\n"
@@ -3838,6 +3847,14 @@ testRun(void)
HRN_STORAGE_PUT(storagePgWrite(), "grow-to-block-incr", file, .timeModified = backupTimeStart);
// Grow file past the bundle limit. The prior size is within the limit but the file must be stored unbundled since the
// original size is what will be copied.
file = bufNew(BLOCK_MIN_FILE_SIZE * 2);
memset(bufPtr(file), 88, bufSize(file));
bufUsedSet(file, bufSize(file));
HRN_STORAGE_PUT(storagePgWrite(), "grow-past-bundle-limit", file, .timeModified = backupTimeStart);
// File that gets truncated to zero during the backup
HRN_STORAGE_PUT(storagePgWrite(), "truncate-to-zero", BUFSTRDEF("DATA"), .timeModified = backupTimeStart);
@@ -3884,6 +3901,7 @@ testRun(void)
"P00 INFO: backup '20191103-165320F_20191106-002640D' cannot be resumed: resume only valid for full backup\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-past-bundle-limit (32KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: store truncated file " TEST_PATH "/pg1/block-incr-truncate (24KB->0B, [PCT])\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/block-incr-same (16KB, [PCT]) checksum [SHA1]\n"
@@ -3901,7 +3919,7 @@ testRun(void)
"P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n"
"P00 INFO: check archive for segment(s) 0000000105DC213000000000:0000000105DC213000000001\n"
"P00 INFO: new backup label = 20191103-165320F_20191106-002640D\n"
"P00 INFO: diff backup size = [SIZE], file total = 13");
"P00 INFO: diff backup size = [SIZE], file total = 14");
TEST_RESULT_STR_Z(
testBackupValidateP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
@@ -3914,6 +3932,7 @@ testRun(void)
"pg_data/backup_label {s=17, ts=+2}\n"
"pg_data/block-incr-grow.pgbi {s=131072, m=0:{0},1:{0},0:{2},1:{1,2,3,4,5,6,7,8,9,10,11,12,13}}\n"
"pg_data/block-incr-larger.pgbi {s=1507328, m=1:{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15},1:{0,1,2,3,4,5,6}}\n"
"pg_data/grow-past-bundle-limit.pgbi {s=32768, m=0:{0,1},1:{0,1}}\n"
"20191103-165320F/bundle/1/pg_data/PG_VERSION {s=2, ts=-200000}\n"
"20191103-165320F/bundle/1/pg_data/block-incr-same {s=16384, m=0:{0,1}}\n"
"20191103-165320F/bundle/1/pg_data/normal-same {s=4}\n"
@@ -3929,6 +3948,7 @@ testRun(void)
HRN_STORAGE_REMOVE(storagePgWrite(), "block-incr-shrink-below");
HRN_STORAGE_REMOVE(storagePgWrite(), "block-incr-shrink-block");
HRN_STORAGE_REMOVE(storagePgWrite(), "block-incr-truncate");
HRN_STORAGE_REMOVE(storagePgWrite(), "grow-past-bundle-limit");
HRN_STORAGE_REMOVE(storagePgWrite(), "grow-to-block-incr");
HRN_STORAGE_REMOVE(storagePgWrite(), "normal-same");
HRN_STORAGE_REMOVE(storagePgWrite(), "truncate-to-zero");