1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-08-18 20:41:58 +03:00

Storage and IO layer refactor:

Refactor storage layer to allow for new repository filesystems using drivers. (Reviewed by Cynthia Shang.)
Refactor IO layer to allow for new compression formats, checksum types, and other capabilities using filters. (Reviewed by Cynthia Shang.)
This commit is contained in:
David Steele
2017-06-09 17:51:41 -04:00
parent 7e982f05f5
commit de7fc37f88
183 changed files with 17880 additions and 14734 deletions

View File

@@ -11,14 +11,13 @@ use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::ArchiveCommon;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -95,58 +94,54 @@ sub run
################################################################################################################################
if ($self->begin("${strModule}::walSegmentFind()"))
{
my $strArchiveId = '9.4-1';
my $oFile = new pgBackRest::File(
$self->stanza(),
$self->testPath(),
new pgBackRest::Protocol::Common::Common(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
));
my $oOption = {};
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->testPath());
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
my $strArchivePath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId);;
my $strArchiveId = '9.4-1';
my $strArchivePath = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegment = '000000010000000100000001ZZ';
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ASSERT, "${strWalSegment} is not a WAL segment");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ASSERT,
"${strWalSegment} is not a WAL segment");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = '000000010000000100000001';
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
"could not find WAL segment ${strWalSegment} after 0.1 second(s)");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
my $strWalSegmentHash = "${strWalSegment}-53aa5d59515aa7288ae02ba414c009aed1ca73ad";
filePathCreate($strWalMajorPath, undef, false, true);
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
sub {walSegmentFind(storageRepo(), $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
"${strWalSegment} WAL found without timeline");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegmentHash2 = "${strWalSegment}-a0b0d38b8aa263e25b8ff52a0a4ba85b6be97f9b.gz";
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash2}");
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash2}");
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
"duplicates found in archive for WAL segment ${strWalSegment}: ${strWalSegmentHash}, ${strWalSegmentHash2}");
#---------------------------------------------------------------------------------------------------------------------------
@@ -154,31 +149,31 @@ sub run
my $strWalSegmentHash3 = "${strWalSegment3}-dcdd09246e1918e88c67cf44b35edc23b803d879";
my $strWalMajorPath3 = "${strArchivePath}/" . substr($strWalSegment3, 0, 16);
filePathCreate($strWalMajorPath3, undef, false, true);
fileStringWrite("${strWalMajorPath3}/${strWalSegmentHash3}");
storageRepo()->pathCreate($strWalMajorPath3, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath3}/${strWalSegmentHash3}");
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
sub {walSegmentFind(storageRepo(), $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
"duplicates found in archive for WAL segment XXXXXXXX" . substr($strWalSegment, 8, 16) .
": ${strWalSegmentHash}, ${strWalSegmentHash2}, ${strWalSegmentHash3}");
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
fileRemove("${strWalMajorPath3}/${strWalSegmentHash3}");
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash}");
storageRepo()->remove("${strWalMajorPath3}/${strWalSegmentHash3}");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
"${strWalSegment} WAL found with compressed extension");
fileRemove("${strWalMajorPath}/${strWalSegmentHash2}");
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash2}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = $strWalSegment . '.partial';
$strWalSegmentHash = "${strWalSegment}-996195c807713ef9262170043e7222cb150aef70";
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
}
}