1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-08-08 14:22:06 +03:00

Rename DeltaMap to BlockHash.

This more accurately describes what the object does.
This commit is contained in:
David Steele
2023-02-13 09:17:30 +07:00
parent 779efe0d7a
commit dffc933384
10 changed files with 59 additions and 58 deletions

View File

@@ -22,6 +22,7 @@
<commit subject="Improve IoChunkedRead end-of-file handling."/> <commit subject="Improve IoChunkedRead end-of-file handling."/>
<commit subject="Remove parameter list from deltaMapNew()."/> <commit subject="Remove parameter list from deltaMapNew()."/>
<commit subject="Consistently declare block incremental size as size_t."/> <commit subject="Consistently declare block incremental size as size_t."/>
<commit subject="Rename DeltaMap to BlockHash."/>
<release-item-contributor-list> <release-item-contributor-list>
<release-item-contributor id="david.steele"/> <release-item-contributor id="david.steele"/>

View File

@@ -82,7 +82,7 @@ SRCS = \
command/repo/ls.c \ command/repo/ls.c \
command/repo/put.c \ command/repo/put.c \
command/repo/rm.c \ command/repo/rm.c \
command/restore/deltaMap.c \ command/restore/blockHash.c \
command/restore/file.c \ command/restore/file.c \
command/restore/protocol.c \ command/restore/protocol.c \
command/restore/restore.c \ command/restore/restore.c \

View File

@@ -3,7 +3,7 @@ Restore Delta Map
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#include "build.auto.h" #include "build.auto.h"
#include "command/restore/deltaMap.h" #include "command/restore/blockHash.h"
#include "common/crypto/common.h" #include "common/crypto/common.h"
#include "common/crypto/hash.h" #include "common/crypto/hash.h"
#include "common/debug.h" #include "common/debug.h"
@@ -13,34 +13,34 @@ Restore Delta Map
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Object type Object type
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
typedef struct DeltaMap typedef struct BlockHash
{ {
MemContext *memContext; // Mem context of filter MemContext *memContext; // Mem context of filter
size_t blockSize; // Block size for checksums size_t blockSize; // Block size for checksums
size_t blockCurrent; // Size of current block size_t blockCurrent; // Size of current block
IoFilter *hash; // Hash of current block IoFilter *hash; // Hash of current block
List *list; // List if hashes List *list; // List of hashes
} DeltaMap; } BlockHash;
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Macros for function logging Macros for function logging
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#define FUNCTION_LOG_DELTA_MAP_TYPE \ #define FUNCTION_LOG_BLOCK_HASH_TYPE \
DeltaMap * BlockHash *
#define FUNCTION_LOG_DELTA_MAP_FORMAT(value, buffer, bufferSize) \ #define FUNCTION_LOG_BLOCK_HASH_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "DeltaMap", buffer, bufferSize) objNameToLog(value, "BlockHash", buffer, bufferSize)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Generate delta map Generate block hash list
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
static void static void
deltaMapProcess(THIS_VOID, const Buffer *const input) blockHashProcess(THIS_VOID, const Buffer *const input)
{ {
THIS(DeltaMap); THIS(BlockHash);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(DELTA_MAP, this); FUNCTION_LOG_PARAM(BLOCK_HASH, this);
FUNCTION_LOG_PARAM(BUFFER, input); FUNCTION_LOG_PARAM(BUFFER, input);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
@@ -95,12 +95,12 @@ deltaMapProcess(THIS_VOID, const Buffer *const input)
Get a binary representation of the hash list Get a binary representation of the hash list
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
static Pack * static Pack *
deltaMapResult(THIS_VOID) blockHashResult(THIS_VOID)
{ {
THIS(DeltaMap); THIS(BlockHash);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(DELTA_MAP, this); FUNCTION_LOG_PARAM(BLOCK_HASH, this);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
@@ -127,7 +127,7 @@ deltaMapResult(THIS_VOID)
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
FN_EXTERN IoFilter * FN_EXTERN IoFilter *
deltaMapNew(const size_t blockSize) blockHashNew(const size_t blockSize)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(SIZE, blockSize); FUNCTION_LOG_PARAM(SIZE, blockSize);
@@ -138,18 +138,18 @@ deltaMapNew(const size_t blockSize)
// Allocate memory to hold process state // Allocate memory to hold process state
IoFilter *this = NULL; IoFilter *this = NULL;
OBJ_NEW_BEGIN(DeltaMap, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) OBJ_NEW_BEGIN(BlockHash, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
{ {
DeltaMap *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::DeltaMap); BlockHash *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::BlockHash);
*driver = (DeltaMap) *driver = (BlockHash)
{ {
.memContext = memContextCurrent(), .memContext = memContextCurrent(),
.blockSize = blockSize, .blockSize = blockSize,
.list = lstNewP(HASH_TYPE_SHA1_SIZE), .list = lstNewP(HASH_TYPE_SHA1_SIZE),
}; };
this = ioFilterNewP(DELTA_MAP_FILTER_TYPE, driver, NULL, .in = deltaMapProcess, .result = deltaMapResult); this = ioFilterNewP(BLOCK_HASH_FILTER_TYPE, driver, NULL, .in = blockHashProcess, .result = blockHashResult);
} }
OBJ_NEW_END(); OBJ_NEW_END();

View File

@@ -1,11 +1,11 @@
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Restore Delta Map Block Hash List
Build a list of hashes based on a block size. This is used to compare the contents of a file to block map to determine what needs to Build a list of hashes based on a block size. This is used to compare the contents of a file to block map to determine what needs to
be updated. be updated.
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#ifndef COMMAND_RESTORE_DELTA_MAP_H #ifndef COMMAND_RESTORE_BLOCK_HASH_H
#define COMMAND_RESTORE_DELTA_MAP_H #define COMMAND_RESTORE_BLOCK_HASH_H
#include "common/io/filter/filter.h" #include "common/io/filter/filter.h"
#include "common/type/stringId.h" #include "common/type/stringId.h"
@@ -13,11 +13,11 @@ be updated.
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Filter type constant Filter type constant
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#define DELTA_MAP_FILTER_TYPE STRID5("dlt-map", 0x402ddd1840) #define BLOCK_HASH_FILTER_TYPE STRID5("dlt-map", 0x402ddd1840)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructors Constructors
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
FN_EXTERN IoFilter *deltaMapNew(size_t blockSize); FN_EXTERN IoFilter *blockHashNew(size_t blockSize);
#endif #endif

View File

@@ -8,7 +8,7 @@ Restore File
#include <utime.h> #include <utime.h>
#include "command/backup/blockMap.h" #include "command/backup/blockMap.h"
#include "command/restore/deltaMap.h" #include "command/restore/blockHash.h"
#include "command/restore/file.h" #include "command/restore/file.h"
#include "common/crypto/cipherBlock.h" #include "common/crypto/cipherBlock.h"
#include "common/crypto/hash.h" #include "common/crypto/hash.h"
@@ -122,9 +122,9 @@ restoreFile(
read = storageReadIo(storageNewReadP(storagePg(), file->name)); read = storageReadIo(storageNewReadP(storagePg(), file->name));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1)); ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
// Generate delta map if block incremental // Generate block hash list if block incremental
if (file->blockIncrMapSize != 0) if (file->blockIncrMapSize != 0)
ioFilterGroupAdd(ioReadFilterGroup(read), deltaMapNew(file->blockIncrSize)); ioFilterGroupAdd(ioReadFilterGroup(read), blockHashNew(file->blockIncrSize));
ioReadDrain(read); ioReadDrain(read);
} }
@@ -154,16 +154,16 @@ restoreFile(
fileResult->result = restoreResultPreserve; fileResult->result = restoreResultPreserve;
} }
// If block incremental and not preserving the file, store the delta map for later use in // If block incremental and not preserving the file, store the block hash list for later use in
// reconstructing the pg file // reconstructing the pg file
if (file->blockIncrMapSize != 0 && fileResult->result != restoreResultPreserve) if (file->blockIncrMapSize != 0 && fileResult->result != restoreResultPreserve)
{ {
PackRead *const deltaMapResult = ioFilterGroupResultP( PackRead *const blockHashResult = ioFilterGroupResultP(
ioReadFilterGroup(read), DELTA_MAP_FILTER_TYPE); ioReadFilterGroup(read), BLOCK_HASH_FILTER_TYPE);
MEM_CONTEXT_OBJ_BEGIN(fileList) MEM_CONTEXT_OBJ_BEGIN(fileList)
{ {
file->deltaMap = pckReadBinP(deltaMapResult); file->blockHash = pckReadBinP(blockHashResult);
} }
MEM_CONTEXT_OBJ_END(); MEM_CONTEXT_OBJ_END();
} }
@@ -277,7 +277,7 @@ restoreFile(
StorageWrite *pgFileWrite = storageNewWriteP( StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group, storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true, .timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true,
.noTruncate = file->deltaMap != NULL); .noTruncate = file->blockHash != NULL);
// If block incremental file // If block incremental file
const Buffer *checksum = NULL; const Buffer *checksum = NULL;
@@ -286,17 +286,17 @@ restoreFile(
{ {
ASSERT(referenceList != NULL); ASSERT(referenceList != NULL);
// Read block map. This will be compared to the delta map already created to determine which blocks need to // Read block map. This will be compared to the block hash list already created to determine which blocks
// be fetched from the repository. If we got here there must be at least one block to fetch. // need to be fetched from the repository. If we got here there must be at least one block to fetch.
const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead)); const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead));
// The repo file needs to be closed so that block lists can be read from the remote protocol // The repo file needs to be closed so that block lists can be read from the remote protocol
ioReadClose(storageReadIo(repoFileRead)); ioReadClose(storageReadIo(repoFileRead));
// Size of delta map. If there is no delta map because the pg file does not exist then set to zero, which // Size of block hash list. If there is no block hash list because the pg file does not exist then set to
// will force all blocks to be updated. // zero, which will force all blocks to be updated.
const unsigned int deltaMapSize = const unsigned int blockHashSize =
file->deltaMap == NULL ? 0 : (unsigned int)(bufUsed(file->deltaMap) / HASH_TYPE_SHA1_SIZE); file->blockHash == NULL ? 0 : (unsigned int)(bufUsed(file->blockHash) / HASH_TYPE_SHA1_SIZE);
// Find and write updated blocks // Find and write updated blocks
bool updateFound = false; // Is there a block list to be updated? bool updateFound = false; // Is there a block list to be updated?
@@ -311,12 +311,12 @@ restoreFile(
{ {
const BlockMapItem *const blockMapItem = blockMapGet(blockMap, blockMapIdx); const BlockMapItem *const blockMapItem = blockMapGet(blockMap, blockMapIdx);
// The block must be updated if it beyond the blocks that exist in the delta map or when the checksum // The block must be updated if it beyond the blocks that exist in the block hash list or when the
// stored in the repository is different from the delta map // checksum stored in the repository is different from the block hash list
if (blockMapIdx >= deltaMapSize || if (blockMapIdx >= blockHashSize ||
!bufEq( !bufEq(
BUF(blockMapItem->checksum, HASH_TYPE_SHA1_SIZE), BUF(blockMapItem->checksum, HASH_TYPE_SHA1_SIZE),
BUF(bufPtrConst(file->deltaMap) + blockMapIdx * HASH_TYPE_SHA1_SIZE, HASH_TYPE_SHA1_SIZE))) BUF(bufPtrConst(file->blockHash) + blockMapIdx * HASH_TYPE_SHA1_SIZE, HASH_TYPE_SHA1_SIZE)))
{ {
// If no block list is currently being built then start a new one // If no block list is currently being built then start a new one
if (!updateFound) if (!updateFound)
@@ -343,11 +343,11 @@ restoreFile(
// Similar to the check above, but also make sure the reference is the same. For blocks to be // Similar to the check above, but also make sure the reference is the same. For blocks to be
// in a common list they must be contiguous and from the same reference. // in a common list they must be contiguous and from the same reference.
if (blockMapItem->reference == blockMapItemNext->reference && if (blockMapItem->reference == blockMapItemNext->reference &&
(blockMapIdx + 1 >= deltaMapSize || (blockMapIdx + 1 >= blockHashSize ||
!bufEq( !bufEq(
BUF(blockMapItemNext->checksum, HASH_TYPE_SHA1_SIZE), BUF(blockMapItemNext->checksum, HASH_TYPE_SHA1_SIZE),
BUF( BUF(
bufPtrConst(file->deltaMap) + (blockMapIdx + 1) * HASH_TYPE_SHA1_SIZE, bufPtrConst(file->blockHash) + (blockMapIdx + 1) * HASH_TYPE_SHA1_SIZE,
HASH_TYPE_SHA1_SIZE)))) HASH_TYPE_SHA1_SIZE))))
{ {
continue; continue;

View File

@@ -36,7 +36,7 @@ typedef struct RestoreFile
uint64_t blockIncrMapSize; // Block incremental map size (0 if not incremental) uint64_t blockIncrMapSize; // Block incremental map size (0 if not incremental)
size_t blockIncrSize; // Block incremental size (when map size > 0) size_t blockIncrSize; // Block incremental size (when map size > 0)
const String *manifestFile; // Manifest file const String *manifestFile; // Manifest file
const Buffer *deltaMap; // Delta for block incremental restore, set in restoreFile() const Buffer *blockHash; // Hashes for block incremental restore, set in restoreFile()
} RestoreFile; } RestoreFile;
typedef struct RestoreFileResult typedef struct RestoreFileResult

View File

@@ -146,7 +146,7 @@ src_pgbackrest = [
'command/repo/ls.c', 'command/repo/ls.c',
'command/repo/put.c', 'command/repo/put.c',
'command/repo/rm.c', 'command/repo/rm.c',
'command/restore/deltaMap.c', 'command/restore/blockHash.c',
'command/restore/file.c', 'command/restore/file.c',
'command/restore/protocol.c', 'command/restore/protocol.c',
'command/restore/restore.c', 'command/restore/restore.c',

View File

@@ -575,11 +575,11 @@ src/command/repo/rm.h:
class: core class: core
type: c/h type: c/h
src/command/restore/deltaMap.c: src/command/restore/blockHash.c:
class: core class: core
type: c type: c
src/command/restore/deltaMap.h: src/command/restore/blockHash.h:
class: core class: core
type: c/h type: c/h

View File

@@ -843,7 +843,7 @@ unit:
total: 14 total: 14
coverage: coverage:
- command/restore/deltaMap - command/restore/blockHash
- command/restore/file - command/restore/file
- command/restore/protocol - command/restore/protocol
- command/restore/restore - command/restore/restore

View File

@@ -157,14 +157,14 @@ testRun(void)
Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true); Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("DeltaMap")) if (testBegin("BlockHash"))
{ {
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("too large for one buffer"); TEST_TITLE("too large for one buffer");
Buffer *output = bufNew(0); Buffer *output = bufNew(0);
IoWrite *write = ioBufferWriteNew(output); IoWrite *write = ioBufferWriteNew(output);
ioFilterGroupAdd(ioWriteFilterGroup(write), deltaMapNew(3)); ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3));
ioWriteOpen(write); ioWriteOpen(write);
TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("ABCDEF")), "write"); TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("ABCDEF")), "write");
@@ -172,18 +172,18 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close"); TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), DELTA_MAP_FILTER_TYPE))), strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8" "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
"6dae29c06c5f04601445c493156d10fe1be23b6d" "6dae29c06c5f04601445c493156d10fe1be23b6d"
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8", "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8",
"delta map"); "block hash list");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("buffer smaller than block and remainder"); TEST_TITLE("buffer smaller than block and remainder");
output = bufNew(0); output = bufNew(0);
write = ioBufferWriteNew(output); write = ioBufferWriteNew(output);
ioFilterGroupAdd(ioWriteFilterGroup(write), deltaMapNew(3)); ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3));
ioWriteOpen(write); ioWriteOpen(write);
TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("DE")), "write"); TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("DE")), "write");
@@ -194,12 +194,12 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close"); TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), DELTA_MAP_FILTER_TYPE))), strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
"6dae29c06c5f04601445c493156d10fe1be23b6d" "6dae29c06c5f04601445c493156d10fe1be23b6d"
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8" "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8" "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
"c032adc1ff629c9b66f22749ad667e6beadf144b", "c032adc1ff629c9b66f22749ad667e6beadf144b",
"delta map"); "block hash list");
} }
// ***************************************************************************************************************************** // *****************************************************************************************************************************