+
+
+
+
+
+ Refactor storage layer to allow for new repository filesystems using drivers.
+
+
+
+
+
+
+
+ Refactor IO layer to allow for new compression formats, checksum types, and other capabilities using filters.
+
+
Move modules in Protocol directory in subdirectories.
diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml
index e6ee55623..c469e8423 100644
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@@ -81,7 +81,7 @@
{[host-mount]}
- ls -1 {[backrest-repo-path]}/backup/demo | tail -4 | head -1
+ ls -1 {[backrest-repo-path]}/backup/demo | tail -5 | head -1
Important Data
@@ -697,7 +697,7 @@
Revoke write privileges in the <backrest/> repository and attempt a backup
- chmod 550 {[backrest-repo-path]}/temp
+ chmod 550 {[backrest-repo-path]}/backup/{[postgres-cluster-demo]}/
@@ -713,7 +713,7 @@
Restore write privileges in the <backrest/> repository and attempt a backup
- chmod 750 {[backrest-repo-path]}/temp
+ chmod 750 {[backrest-repo-path]}/backup/{[postgres-cluster-demo]}/
@@ -1384,8 +1384,6 @@
{[db-path]}
- {[backrest-repo-path]}
-
{[host-backup]}
backrest
@@ -1634,7 +1632,7 @@
{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} backup
- remote process terminated on [^ ]+ host: stop file exists for all stanzas
+ process .* terminated unexpectedly.*: stop file exists for all stanzas
@@ -1675,7 +1673,7 @@
{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} backup
- remote process terminated on [^ ]+ host: stop file exists for stanza demo
+ process .* terminated unexpectedly.*: stop file exists for stanza demo
@@ -1759,7 +1757,6 @@
{[db-path]}
- {[backrest-repo-path]}
{[host-backup]}
standby_mode=on
@@ -2081,7 +2078,7 @@
-
+
Upgrading <postgres/>
diff --git a/lib/pgBackRest/Archive/Archive.pm b/lib/pgBackRest/Archive/Archive.pm
index dd76314b0..33ae8efaf 100644
--- a/lib/pgBackRest/Archive/Archive.pm
+++ b/lib/pgBackRest/Archive/Archive.pm
@@ -19,8 +19,9 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
@@ -64,7 +65,6 @@ sub getCheck
my
(
$strOperation,
- $oFile,
$strDbVersion,
$ullDbSysId,
$strWalFile,
@@ -72,7 +72,6 @@ sub getCheck
logDebugParam
(
__PACKAGE__ . '->getCheck', \@_,
- {name => 'oFile'},
{name => 'strDbVersion', required => false},
{name => 'ullDbSysId', required => false},
{name => 'strWalFile', required => false},
@@ -88,20 +87,22 @@ sub getCheck
($strDbVersion, my $iControlVersion, my $iCatalogVersion, $ullDbSysId) = dbMasterGet()->info();
}
- if ($oFile->isRemote(PATH_BACKUP_ARCHIVE))
+ # Get db info from the repo
+ if (!isRepoLocal())
{
- ($strArchiveId, $strArchiveFile) = $oFile->{oProtocol}->cmdExecute(
+ ($strArchiveId, $strArchiveFile) = protocolGet(BACKUP)->cmdExecute(
OP_ARCHIVE_GET_CHECK, [$strDbVersion, $ullDbSysId, $strWalFile], true);
}
else
{
# check that the archive info is compatible with the database
$strArchiveId =
- (new pgBackRest::Archive::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE), true))->check($strDbVersion, $ullDbSysId);
+ (new pgBackRest::Archive::ArchiveInfo(
+ storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true))->check($strDbVersion, $ullDbSysId);
if (defined($strWalFile))
{
- $strArchiveFile = walSegmentFind($oFile, ${strArchiveId}, $strWalFile);
+ $strArchiveFile = walSegmentFind(storageRepo(), ${strArchiveId}, $strWalFile);
}
}
diff --git a/lib/pgBackRest/Archive/ArchiveCommon.pm b/lib/pgBackRest/Archive/ArchiveCommon.pm
index 7ec34ec55..3b726d155 100644
--- a/lib/pgBackRest/Archive/ArchiveCommon.pm
+++ b/lib/pgBackRest/Archive/ArchiveCommon.pm
@@ -9,7 +9,7 @@ use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
-use Fcntl qw(SEEK_CUR O_RDONLY); # !!! Only needed until read from buffer
+use Fcntl qw(SEEK_CUR O_RDONLY);
use File::Basename qw(dirname);
use pgBackRest::Db;
@@ -18,9 +18,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# RegEx constants
@@ -38,6 +37,12 @@ use constant PG_WAL_SYSTEM_ID_OFFSET_GTE_93 => 20;
use constant PG_WAL_SYSTEM_ID_OFFSET_LT_93 => 12;
push @EXPORT, qw(PG_WAL_SYSTEM_ID_OFFSET_LT_93);
+####################################################################################################################################
+# WAL segment size
+####################################################################################################################################
+use constant PG_WAL_SEGMENT_SIZE => 16777216;
+ push @EXPORT, qw(PG_WAL_SEGMENT_SIZE);
+
####################################################################################################################################
# PostgreSQL WAL magic
####################################################################################################################################
@@ -260,7 +265,7 @@ sub walSegmentFind
my
(
$strOperation,
- $oFile,
+ $oStorageRepo,
$strArchiveId,
$strWalSegment,
$iWaitSeconds,
@@ -268,7 +273,7 @@ sub walSegmentFind
logDebugParam
(
__PACKAGE__ . '::walSegmentFind', \@_,
- {name => 'oFile'},
+ {name => 'oStorageRepo'},
{name => 'strArchiveId'},
{name => 'strWalSegment'},
{name => 'iWaitSeconds', required => false},
@@ -298,8 +303,8 @@ sub walSegmentFind
}
else
{
- @stryTimelineMajor = $oFile->list(
- PATH_BACKUP_ARCHIVE, $strArchiveId,
+ @stryTimelineMajor = $oStorageRepo->list(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveId}",
{strExpression => '[0-F]{8}' . substr($strWalSegment, 0, 8), bIgnoreMissing => true});
}
@@ -310,8 +315,8 @@ sub walSegmentFind
my $strWalSegmentFind = $bTimeline ? substr($strWalSegment, 0, 24) : $strTimelineMajor . substr($strWalSegment, 8, 16);
# Get the name of the requested WAL segment (may have hash info and compression extension)
- push(@stryWalFileName, $oFile->list(
- PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strTimelineMajor}",
+ push(@stryWalFileName, $oStorageRepo->list(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strTimelineMajor}",
{strExpression =>
"^${strWalSegmentFind}" . (walIsPartial($strWalSegment) ? "\\.partial" : '') .
"-[0-f]{40}(\\." . COMPRESS_EXT . "){0,1}\$",
diff --git a/lib/pgBackRest/Archive/ArchiveGet.pm b/lib/pgBackRest/Archive/ArchiveGet.pm
index 3c131db11..270d658e4 100644
--- a/lib/pgBackRest/Archive/ArchiveGet.pm
+++ b/lib/pgBackRest/Archive/ArchiveGet.pm
@@ -25,10 +25,11 @@ use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Db;
use pgBackRest::DbVersion;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Filter::Gzip;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# process
@@ -86,23 +87,18 @@ sub get
lockStopTest();
- # Create the file object
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(BACKUP)
- );
+ # Get the repo storage
+ my $oStorageRepo = storageRepo();
# Construct absolute path to the WAL file when it is relative
$strDestinationFile = walPath($strDestinationFile, optionGet(OPTION_DB_PATH, false), commandGet());
# Get the wal segment filename
my ($strArchiveId, $strArchiveFile) = $self->getCheck(
- $oFile, undef, undef, walIsSegment($strSourceArchive) ? $strSourceArchive : undef);
+ undef, undef, walIsSegment($strSourceArchive) ? $strSourceArchive : undef);
if (!defined($strArchiveFile) && !walIsSegment($strSourceArchive) &&
- $oFile->exists(PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strSourceArchive}"))
+ $oStorageRepo->exists(STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strSourceArchive}"))
{
$strArchiveFile = $strSourceArchive;
}
@@ -122,13 +118,16 @@ sub get
else
{
# Determine if the source file is already compressed
- my $bSourceCompressed = $strArchiveFile =~ "^.*\.$oFile->{strCompressExtension}\$" ? true : false;
+ my $bSourceCompressed = $strArchiveFile =~ ('^.*\.' . COMPRESS_EXT . '$') ? true : false;
# Copy the archive file to the requested location
- $oFile->copy(PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strArchiveFile}", # Source file
- PATH_DB_ABSOLUTE, $strDestinationFile, # Destination file
- $bSourceCompressed, # Source compression based on detection
- false); # Destination is not compressed
+ $oStorageRepo->copy(
+ $oStorageRepo->openRead(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strArchiveFile}", {bProtocolCompress => !$bSourceCompressed}),
+ storageDb()->openWrite(
+ $strDestinationFile,
+ {rhyFilter => $bSourceCompressed ?
+ [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
}
# Return from function and log return values if any
@@ -149,20 +148,19 @@ sub get
sub getArchiveId
{
my $self = shift;
- my $oFile = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->getArchiveId');
my $strArchiveId;
- if ($oFile->isRemote(PATH_BACKUP_ARCHIVE))
+ if (!isRepoLocal())
{
- $strArchiveId = $oFile->{oProtocol}->cmdExecute(OP_ARCHIVE_GET_ARCHIVE_ID, undef, true);
+ $strArchiveId = protocolGet(BACKUP)->cmdExecute(OP_ARCHIVE_GET_ARCHIVE_ID, undef, true);
}
else
{
- $strArchiveId = (new pgBackRest::Archive::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE), true))->archiveId();
+ $strArchiveId = (new pgBackRest::Archive::ArchiveInfo(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true))->archiveId();
}
# Return from function and log return values if any
diff --git a/lib/pgBackRest/Archive/ArchiveInfo.pm b/lib/pgBackRest/Archive/ArchiveInfo.pm
index 540441c50..979ba583b 100644
--- a/lib/pgBackRest/Archive/ArchiveInfo.pm
+++ b/lib/pgBackRest/Archive/ArchiveInfo.pm
@@ -14,27 +14,27 @@ use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
+ our @EXPORT = qw();
use File::Basename qw(dirname basename);
-use File::stat;
-use Fcntl qw(O_RDONLY);
-use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+use pgBackRest::Archive::ArchiveCommon;
use pgBackRest::Common::Exception;
+use pgBackRest::Config::Config;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
-use pgBackRest::Archive::ArchiveCommon;
-use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Filter::Gzip;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# File/path constants
####################################################################################################################################
use constant ARCHIVE_INFO_FILE => 'archive.info';
- our @EXPORT = qw(ARCHIVE_INFO_FILE);
+ push @EXPORT, qw(ARCHIVE_INFO_FILE);
####################################################################################################################################
# Archive info constants
@@ -71,29 +71,57 @@ sub new
my
(
$strOperation,
- $strArchiveClusterPath, # Backup cluster path
- $bRequired # Is archive info required?
+ $strArchiveClusterPath, # Archive cluster path
+ $bRequired, # Is archive info required?
+ $bLoad, # Should the file attempt to be loaded?
+ $bIgnoreMissing, # Don't error on missing files
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strArchiveClusterPath'},
- {name => 'bRequired', default => true}
+ {name => 'bRequired', default => true},
+ {name => 'bLoad', optional => true, default => true},
+ {name => 'bIgnoreMissing', optional => true, default => false},
);
# Build the archive info path/file name
my $strArchiveInfoFile = "${strArchiveClusterPath}/" . ARCHIVE_INFO_FILE;
- my $bExists = fileExists($strArchiveInfoFile);
-
- if (!$bExists && $bRequired)
- {
- confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING);
- }
+ my $self = {};
+ my $iResult = 0;
+ my $strResultMessage;
# Init object and store variables
- my $self = $class->SUPER::new($strArchiveInfoFile, {bLoad => $bExists});
+ eval
+ {
+ $self = $class->SUPER::new($strArchiveInfoFile, {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing,
+ oStorage => storageRepo()});
+ return true;
+ }
+ or do
+ {
+ # Capture error information
+ $iResult = exceptionCode($EVAL_ERROR);
+ $strResultMessage = exceptionMessage($EVAL_ERROR->message());
+ };
+
+ if ($iResult != 0)
+ {
+ # If the file does not exist but is required to exist, then error
+ # The archive info is only allowed not to exist when running a stanza-create on a new install
+ if ($iResult == ERROR_FILE_MISSING)
+ {
+ if ($bRequired)
+ {
+ confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING);
+ }
+ }
+ else
+ {
+ confess &log(ERROR, $strResultMessage, $iResult);
+ }
+ }
- $self->{bExists} = $bExists;
$self->{strArchiveClusterPath} = $strArchiveClusterPath;
# Return from function and log return values if any
@@ -276,21 +304,19 @@ sub reconstruct
my
(
$strOperation,
- $oFile,
$strCurrentDbVersion,
$ullCurrentDbSysId,
) =
logDebugParam
(
__PACKAGE__ . '->reconstruct', \@_,
- {name => 'oFile'},
{name => 'strCurrentDbVersion'},
{name => 'ullCurrentDbSysId'},
);
my $strInvalidFileStructure = undef;
- my @stryArchiveId = fileList(
+ my @stryArchiveId = storageRepo()->list(
$self->{strArchiveClusterPath}, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});
my %hDbHistoryVersion;
@@ -308,7 +334,7 @@ sub reconstruct
my $strVersionDir = $strDbVersion . "-" . $iDbHistoryId;
# Get the name of the first archive directory
- my $strArchiveDir = (fileList(
+ my $strArchiveDir = (storageRepo()->list(
$self->{strArchiveClusterPath} . "/${strVersionDir}",
{strExpression => REGEX_ARCHIVE_DIR_WAL, bIgnoreMissing => true}))[0];
@@ -320,9 +346,9 @@ sub reconstruct
}
# ??? Should probably make a function in ArchiveCommon
- my $strArchiveFile = (fileList(
+ my $strArchiveFile = (storageRepo()->list(
$self->{strArchiveClusterPath} . "/${strVersionDir}/${strArchiveDir}",
- {strExpression => "^[0-F]{24}(\\.partial){0,1}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$",
+ {strExpression => "^[0-F]{24}(\\.partial){0,1}(-[0-f]+){0,1}(\\." . COMPRESS_EXT . "){0,1}\$",
bIgnoreMissing => true}))[0];
# Continue if any file structure or missing files info
@@ -339,27 +365,16 @@ sub reconstruct
# Get the db-system-id from the WAL file depending on the version of postgres
my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93;
- # If the file is a compressed file, unzip it, else open the first 8KB
+ # Read first 8k of WAL segment
my $tBlock;
- sysopen(my $hFile, $strArchiveFilePath, O_RDONLY)
- or confess &log(ERROR, "unable to open ${strArchiveFilePath}", ERROR_FILE_OPEN);
+ my $oFileIo = storageRepo()->openRead(
+ $strArchiveFilePath,
+ {rhyFilter => $strArchiveFile =~ ('\.' . COMPRESS_EXT . '$') ?
+ [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef});
- if ($strArchiveFile =~ "^.*\.$oFile->{strCompressExtension}\$")
- {
- gunzip($hFile => \$tBlock)
- or confess &log(ERROR,
- "gunzip failed with error: " . $GunzipError .
- " on file ${strArchiveFilePath}", ERROR_FILE_READ);
- }
- else
- {
- # Read part of the file
- sysread($hFile, $tBlock, 8192) == 8192
- or confess &log(ERROR, "unable to read ${strArchiveFilePath}", ERROR_FILE_READ);
- }
-
- close($hFile);
+ $oFileIo->read(\$tBlock, 512, true);
+ $oFileIo->close();
# Get the required data from the file that was pulled into scalar $tBlock
my ($iMagic, $iFlag, $junk, $ullDbSysId) = unpack('SSa' . $iSysIdOffset . 'Q', $tBlock);
diff --git a/lib/pgBackRest/Archive/ArchivePush.pm b/lib/pgBackRest/Archive/ArchivePush.pm
index 50d5a490f..6ffa957a5 100644
--- a/lib/pgBackRest/Archive/ArchivePush.pm
+++ b/lib/pgBackRest/Archive/ArchivePush.pm
@@ -20,10 +20,9 @@ use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# WAL status constants
@@ -79,8 +78,7 @@ sub process
if (optionGet(OPTION_ARCHIVE_ASYNC))
{
# Get the spool path
- $self->{strSpoolPath} = (new pgBackRest::File(
- optionGet(OPTION_STANZA), optionGet(OPTION_SPOOL_PATH), protocolGet(NONE)))->pathGet(PATH_BACKUP_ARCHIVE_OUT);
+ $self->{strSpoolPath} = storageSpool()->pathGet(STORAGE_SPOOL_ARCHIVE_OUT);
# Loop to check for status files and launch async process
my $bPushed = false;
@@ -119,14 +117,6 @@ sub process
require pgBackRest::Archive::ArchivePushFile;
pgBackRest::Archive::ArchivePushFile->import();
- # Create the file object
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(BACKUP)
- );
-
# Drop file if queue max has been exceeded
$self->{strWalPath} = $strWalPath;
@@ -138,7 +128,7 @@ sub process
# Else push the WAL file
else
{
- archivePushFile($oFile, $strWalPath, $strWalFile, optionGet(OPTION_COMPRESS), optionGet(OPTION_REPO_SYNC));
+ archivePushFile($strWalPath, $strWalFile, optionGet(OPTION_COMPRESS));
}
}
@@ -185,7 +175,8 @@ sub walStatus
my $bResult = false;
# Find matching status files
- my @stryStatusFile = fileList($strSpoolPath, {strExpression => '^' . $strWalFile . '\.(ok|error)$', bIgnoreMissing => true});
+ my @stryStatusFile = storageSpool()->list(
+ $strSpoolPath, {strExpression => '^' . $strWalFile . '\.(ok|error)$', bIgnoreMissing => true});
if (@stryStatusFile > 0)
{
@@ -197,7 +188,8 @@ sub walStatus
}
# Read the status file
- my @stryWalStatus = split("\n", fileStringRead("${strSpoolPath}/$stryStatusFile[0]"));
+ my $rstrWalStatus = storageSpool()->get("${strSpoolPath}/$stryStatusFile[0]");
+ my @stryWalStatus = split("\n", defined($$rstrWalStatus) ? $$rstrWalStatus : '');
# Status file must have at least two lines if it has content
my $iCode;
@@ -272,7 +264,7 @@ sub readyList
if (defined($self->{strSpoolPath}))
{
- foreach my $strOkFile (fileList($self->{strSpoolPath}, {strExpression => '\.ok$', bIgnoreMissing => true}))
+ foreach my $strOkFile (storageSpool()->list($self->{strSpoolPath}, {strExpression => '\.ok$', bIgnoreMissing => true}))
{
$strOkFile = substr($strOkFile, 0, length($strOkFile) - length('.ok'));
$hOkFile->{$strOkFile} = true;
@@ -281,7 +273,7 @@ sub readyList
# Read the .ready files
my $strWalStatusPath = "$self->{strWalPath}/archive_status";
- my @stryReadyFile = fileList($strWalStatusPath, {strExpression => '\.ready$'});
+ my @stryReadyFile = storageDb()->list($strWalStatusPath, {strExpression => '\.ready$'});
# Generate a list of new files
my @stryNewReadyFile;
@@ -308,7 +300,7 @@ sub readyList
{
if (!defined($hReadyFile->{$strOkFile}))
{
- fileRemove("$self->{strSpoolPath}/${strOkFile}.ok");
+ storageSpool()->remove("$self->{strSpoolPath}/${strOkFile}.ok");
}
}
diff --git a/lib/pgBackRest/Archive/ArchivePushAsync.pm b/lib/pgBackRest/Archive/ArchivePushAsync.pm
index 6f65db87a..225221ac7 100644
--- a/lib/pgBackRest/Archive/ArchivePushAsync.pm
+++ b/lib/pgBackRest/Archive/ArchivePushAsync.pm
@@ -28,11 +28,9 @@ use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Db;
use pgBackRest::DbVersion;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Local::Process;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
@@ -151,7 +149,7 @@ sub initServer
my ($strOperation) = logDebugParam(__PACKAGE__ . '->initServer');
# Create the spool path
- filePathCreate($self->{strSpoolPath}, undef, true, true);
+ storageSpool()->pathCreate($self->{strSpoolPath}, {bIgnoreExists => true, bCreateParent => true});
# Initialize the archive process
$self->{oArchiveProcess} = new pgBackRest::Protocol::Local::Process(
@@ -203,8 +201,7 @@ sub processQueue
foreach my $strWalFile (@{$stryWalFile})
{
$self->{oArchiveProcess}->queueJob(
- 1, 'default', $strWalFile, OP_ARCHIVE_PUSH_FILE,
- [$self->{strWalPath}, $strWalFile, optionGet(OPTION_COMPRESS), optionGet(OPTION_REPO_SYNC)]);
+ 1, 'default', $strWalFile, OP_ARCHIVE_PUSH_FILE, [$self->{strWalPath}, $strWalFile, optionGet(OPTION_COMPRESS)]);
}
# Process jobs if there are any
@@ -218,18 +215,15 @@ sub processQueue
'push ' . @{$stryWalFile} . ' WAL file(s) to archive: ' .
${$stryWalFile}[0] . (@{$stryWalFile} > 1 ? "...${$stryWalFile}[-1]" : ''));
- # Protocol created below so errors are handled correctly
- my $oProtocol;
-
eval
{
# Hold a lock when the repo is remote to be sure no other process is pushing WAL
- $oProtocol = protocolGet(BACKUP);
+ !isRepoLocal() && protocolGet(BACKUP);
while (my $hyJob = $self->{oArchiveProcess}->process())
{
# Send keep alives to protocol
- $oProtocol->keepAlive();
+ protocolKeepAlive();
foreach my $hJob (@{$hyJob})
{
@@ -341,7 +335,7 @@ sub walStatusWrite
if ($strType ne WAL_STATUS_ERROR)
{
# Remove the error file, if any
- fileRemove("$self->{strSpoolPath}/${strWalFile}.error", true);
+ storageSpool()->remove("$self->{strSpoolPath}/${strWalFile}.error", {bIgnoreMissing => true});
}
# Write the status file
@@ -361,7 +355,8 @@ sub walStatusWrite
confess &log(ASSERT, 'error status must have iCode and strMessage set');
}
- fileStringWrite("$self->{strSpoolPath}/${strWalFile}.${strType}", $strStatus);
+ storageSpool()->put(
+ storageSpool()->openWrite("$self->{strSpoolPath}/${strWalFile}.${strType}", {bAtomic => true}), $strStatus);
}
1;
diff --git a/lib/pgBackRest/Archive/ArchivePushFile.pm b/lib/pgBackRest/Archive/ArchivePushFile.pm
index 39b311e22..828c6bc2c 100644
--- a/lib/pgBackRest/Archive/ArchivePushFile.pm
+++ b/lib/pgBackRest/Archive/ArchivePushFile.pm
@@ -10,15 +10,18 @@ use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
-use File::Basename qw(basename);
+use File::Basename qw(basename dirname);
use pgBackRest::Archive::ArchiveCommon;
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Filter::Gzip;
+use pgBackRest::Storage::Filter::Sha;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# archivePushCheck
@@ -32,7 +35,6 @@ sub archivePushCheck
my
(
$strOperation,
- $oFile,
$strArchiveFile,
$strDbVersion,
$ullDbSysId,
@@ -41,7 +43,6 @@ sub archivePushCheck
logDebugParam
(
__PACKAGE__ . '::archivePushCheck', \@_,
- {name => 'oFile'},
{name => 'strArchiveFile'},
{name => 'strDbVersion', required => false},
{name => 'ullDbSysId', required => false},
@@ -49,16 +50,17 @@ sub archivePushCheck
);
# Set operation and debug strings
+ my $oStorageRepo = storageRepo();
my $strArchiveId;
my $strChecksum;
# WAL file is segment?
my $bWalSegment = walIsSegment($strArchiveFile);
- if ($oFile->isRemote(PATH_BACKUP_ARCHIVE))
+ if (!isRepoLocal())
{
# Execute the command
- ($strArchiveId, $strChecksum) = $oFile->{oProtocol}->cmdExecute(
+ ($strArchiveId, $strChecksum) = protocolGet(BACKUP)->cmdExecute(
OP_ARCHIVE_PUSH_CHECK, [$strArchiveFile, $strDbVersion, $ullDbSysId], true);
}
else
@@ -67,11 +69,11 @@ sub archivePushCheck
if ($bWalSegment)
{
# If the info file exists check db version and system-id else error
- $strArchiveId = (new pgBackRest::Archive::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE)))->check(
- $strDbVersion, $ullDbSysId);
+ $strArchiveId = (new pgBackRest::Archive::ArchiveInfo(
+ $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE)))->check($strDbVersion, $ullDbSysId);
# Check if the WAL segment already exists in the archive
- my $strFoundFile = walSegmentFind($oFile, $strArchiveId, $strArchiveFile);
+ my $strFoundFile = walSegmentFind($oStorageRepo, $strArchiveId, $strArchiveFile);
if (defined($strFoundFile))
{
@@ -81,7 +83,7 @@ sub archivePushCheck
# Else just get the archive id
else
{
- $strArchiveId = (new pgBackRest::Archive::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE)))->archiveId();
+ $strArchiveId = (new pgBackRest::Archive::ArchiveInfo($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE)))->archiveId();
}
}
@@ -89,7 +91,7 @@ sub archivePushCheck
if (defined($strChecksum) && !commandTest(CMD_REMOTE))
{
- my $strChecksumNew = $oFile->hash(PATH_DB_ABSOLUTE, $strWalFile);
+ my ($strChecksumNew) = storageDb()->hashSize($strWalFile);
if ($strChecksumNew ne $strChecksum)
{
@@ -126,23 +128,20 @@ sub archivePushFile
my
(
$strOperation,
- $oFile,
$strWalPath,
$strWalFile,
$bCompress,
- $bRepoSync,
) =
logDebugParam
(
__PACKAGE__ . '::archivePushFile', \@_,
- {name => 'oFile'},
{name => 'strWalPath'},
{name => 'strWalFile'},
{name => 'bCompress'},
- {name => 'bRepoSync'},
);
# Get cluster info from the WAL
+ my $oStorageRepo = storageRepo();
my $strDbVersion;
my $ullDbSysId;
@@ -153,7 +152,7 @@ sub archivePushFile
# Check if the WAL already exists in the repo
my ($strArchiveId, $strChecksum, $strWarning) = archivePushCheck(
- $oFile, $strWalFile, $strDbVersion, $ullDbSysId, walIsSegment($strWalFile) ? "${strWalPath}/${strWalFile}" : undef);
+ $strWalFile, $strDbVersion, $ullDbSysId, walIsSegment($strWalFile) ? "${strWalPath}/${strWalFile}" : undef);
# Only copy the WAL segment if checksum is not defined. If checksum is defined it means that the WAL segment already exists
# in the repository with the same checksum (else there would have been an error on checksum mismatch).
@@ -161,26 +160,30 @@ sub archivePushFile
{
my $strArchiveFile = "${strArchiveId}/${strWalFile}";
- # Append compression extension
- if (walIsSegment($strWalFile) && $bCompress)
+ # If a WAL segment
+ if (walIsSegment($strWalFile))
{
- $strArchiveFile .= '.' . $oFile->{strCompressExtension};
+ # Get hash
+ my ($strSourceHash) = storageDb()->hashSize("${strWalPath}/${strWalFile}");
+
+ $strArchiveFile .= "-${strSourceHash}";
+
+ # Add compress extension
+ if ($bCompress)
+ {
+ $strArchiveFile .= qw{.} . COMPRESS_EXT;
+ }
}
- # Copy the WAL segment
- $oFile->copy(
- PATH_DB_ABSOLUTE, "${strWalPath}/${strWalFile}", # Source type/file
- PATH_BACKUP_ARCHIVE, $strArchiveFile, # Destination type/file
- false, # Source is not compressed
- walIsSegment($strWalFile) && $bCompress, # Destination compress is configurable
- undef, undef, undef, # Unused params
- true, # Create path if it does not exist
- undef, undef, # Default User and group
- walIsSegment($strWalFile), # Append checksum if WAL segment
- $bRepoSync); # Sync repo directories?
+ # Copy
+ $oStorageRepo->copy(
+ storageDb()->openRead("${strWalPath}/${strWalFile}",
+ {rhyFilter => walIsSegment($strWalFile) && $bCompress ? [{strClass => STORAGE_FILTER_GZIP}] : undef}),
+ $oStorageRepo->openWrite(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveFile}",
+ {bPathCreate => true, bAtomic => true, bProtocolCompress => !walIsSegment($strWalFile) || !$bCompress}));
}
-
# Return from function and log return values if any
return logDebugReturn
(
diff --git a/lib/pgBackRest/Backup/Backup.pm b/lib/pgBackRest/Backup/Backup.pm
index 33ebb1e21..60ee32b05 100644
--- a/lib/pgBackRest/Backup/Backup.pm
+++ b/lib/pgBackRest/Backup/Backup.pm
@@ -9,30 +9,32 @@ use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
-use Fcntl 'SEEK_CUR';
use File::Basename;
-use File::Path qw(remove_tree);
+use pgBackRest::Archive::ArchiveCommon;
+use pgBackRest::Archive::ArchiveGet;
+use pgBackRest::Backup::Filter::PageChecksum; # ??? Temporary until isLibC is moved to a better place
+use pgBackRest::Backup::Common;
+use pgBackRest::Backup::File;
+use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Exit;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
-use pgBackRest::Archive::ArchiveGet;
-use pgBackRest::Archive::ArchiveCommon;
-use pgBackRest::Backup::Common;
-use pgBackRest::Backup::File;
-use pgBackRest::Backup::Info;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
use pgBackRest::Db;
use pgBackRest::DbVersion;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::Manifest;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Local::Process;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Common::Io::Handle;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Filter::Gzip;
+use pgBackRest::Storage::Filter::Sha;
+use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
@@ -58,11 +60,9 @@ sub new
}
####################################################################################################################################
-# fileNotInManifest
-#
-# Find all files in a backup path that are not in the supplied manifest.
+# resumeClean - cleans the directory from a previous failed backup so it can be reused
####################################################################################################################################
-sub fileNotInManifest
+sub resumeClean
{
my $self = shift;
@@ -70,32 +70,35 @@ sub fileNotInManifest
my
(
$strOperation,
- $oFileLocal,
- $strPathType,
+ $oStorageRepo,
+ $strBackupLabel,
$oManifest,
$oAbortedManifest
) =
logDebugParam
(
- __PACKAGE__ . '->fileNotInManifest', \@_,
- {name => 'oFileLocal', trace => true},
- {name => 'strPathType', trace => true},
- {name => 'oManifest', trace => true},
- {name => 'oAbortedManifest', trace => true}
+ __PACKAGE__ . '->resumeClean', \@_,
+ {name => 'oStorageRepo'},
+ {name => 'strBackupLabel'},
+ {name => 'oManifest'},
+ {name => 'oAbortedManifest'}
);
- # Build manifest for aborted temp path
- my $hFile = $oFileLocal->manifest($strPathType);
+ &log(DETAIL, 'clean resumed backup path: ' . $oStorageRepo->pathGet(STORAGE_REPO_BACKUP . "/${strBackupLabel}"));
+
+ # Build manifest for aborted backup path
+ my $hFile = $oStorageRepo->manifest(STORAGE_REPO_BACKUP . "/${strBackupLabel}");
# Get compress flag
my $bCompressed = $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS);
+ # Find paths and files to delete
my @stryFile;
foreach my $strName (sort(keys(%{$hFile})))
{
- # Ignore certain files that will never be in the manifest
- if ($strName eq FILE_MANIFEST ||
+ # Ignore files that will never be in the manifest but should be preserved
+ if ($strName eq FILE_MANIFEST_COPY ||
$strName eq '.')
{
next;
@@ -157,68 +160,26 @@ sub fileNotInManifest
}
}
- # Push the file/path/link to be deleted into the result array
- push @stryFile, $strName;
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'stryFile', value => \@stryFile}
- );
-}
-
-####################################################################################################################################
-# tmpClean
-#
-# Cleans the temp directory from a previous failed backup so it can be reused
-####################################################################################################################################
-sub tmpClean
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $oFileLocal,
- $oManifest,
- $oAbortedManifest
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->tmpClean', \@_,
- {name => 'oFileLocal', trace => true},
- {name => 'oManifest', trace => true},
- {name => 'oAbortedManifest', trace => true}
- );
-
- &log(DETAIL, 'clean backup temp path: ' . $oFileLocal->pathGet(PATH_BACKUP_TMP));
-
- # Get the list of files that should be deleted from temp
- my @stryFile = $self->fileNotInManifest($oFileLocal, PATH_BACKUP_TMP, $oManifest, $oAbortedManifest);
-
- foreach my $strFile (sort {$b cmp $a} @stryFile)
- {
- my $strDelete = $oFileLocal->pathGet(PATH_BACKUP_TMP, $strFile);
-
- # If a path then delete it, all the files should have already been deleted since we are going in reverse order
- if (!-X $strDelete && -d $strDelete)
+ # If a directory then remove it
+ if ($cType eq 'd')
{
- logDebugMisc($strOperation, "remove path ${strDelete}");
-
- rmdir($strDelete)
- or confess &log(ERROR, "unable to delete path ${strDelete}, is it empty?", ERROR_PATH_REMOVE);
+ logDebugMisc($strOperation, "remove path ${strName}");
+ $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strName}", {bRecurse => true});
}
- # Else delete a file
+ # Else add the file/link to be deleted later
else
{
- logDebugMisc($strOperation, "remove file ${strDelete}");
- fileRemove($strDelete);
+ logDebugMisc($strOperation, "remove file ${strName}");
+ push(@stryFile, STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strName}");
}
}
+ # Delete files in batch for more efficiency
+ if (@stryFile > 0)
+ {
+ $oStorageRepo->remove(\@stryFile);
+ }
+
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
@@ -237,7 +198,6 @@ sub processManifest
my
(
$strOperation,
- $oFileMaster,
$strDbMasterPath,
$strDbCopyPath,
$strType,
@@ -245,12 +205,12 @@ sub processManifest
$bCompress,
$bHardLink,
$oBackupManifest,
+ $strBackupLabel,
$strLsnStart,
) =
logDebugParam
(
__PACKAGE__ . '->processManifest', \@_,
- {name => 'oFileMaster'},
{name => 'strDbMasterPath'},
{name => 'strDbCopyPath'},
{name => 'strType'},
@@ -258,6 +218,7 @@ sub processManifest
{name => 'bCompress'},
{name => 'bHardLink'},
{name => 'oBackupManifest'},
+ {name => 'strBackupLabel'},
{name => 'strLsnStart', required => false},
);
@@ -265,8 +226,9 @@ sub processManifest
&log(TEST, TEST_BACKUP_START);
# Get the master protocol for keep-alive
- my $oProtocolMaster = protocolGet(DB, $self->{iMasterRemoteIdx});
- $oProtocolMaster->noOp();
+ my $oProtocolMaster =
+ !isDbLocal({iRemoteIdx => $self->{iMasterRemoteIdx}}) ? protocolGet(DB, $self->{iMasterRemoteIdx}) : undef;
+ defined($oProtocolMaster) && $oProtocolMaster->noOp();
# Initialize the backup process
my $oBackupProcess = new pgBackRest::Protocol::Local::Process(DB);
@@ -288,17 +250,19 @@ sub processManifest
# Create paths
foreach my $strPath ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_PATH))
{
- $oFileMaster->pathCreate(PATH_BACKUP_TMP, $strPath, undef, true);
+ storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strPath}", {bIgnoreExists => true});
}
- if (optionGet(OPTION_REPO_LINK))
+ if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK))
{
for my $strTarget ($oBackupManifest->keys(MANIFEST_SECTION_BACKUP_TARGET))
{
if ($oBackupManifest->isTargetTablespace($strTarget))
{
- $oFileMaster->linkCreate(
- PATH_BACKUP_TMP, $strTarget, PATH_BACKUP_TMP, MANIFEST_TARGET_PGDATA . "/${strTarget}", false, true);
+ storageRepo()->linkCreate(
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strTarget}",
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/" . MANIFEST_TARGET_PGDATA . "/${strTarget}",
+ {bRelative => true});
}
}
}
@@ -328,8 +292,10 @@ sub processManifest
{
logDebugMisc($strOperation, "hardlink ${strRepoFile} to ${strReference}");
- $oFileMaster->linkCreate(
- PATH_BACKUP_CLUSTER, "${strReference}/${strRepoFile}", PATH_BACKUP_TMP, "${strRepoFile}", true, false, true);
+ storageRepo()->linkCreate(
+ STORAGE_REPO_BACKUP . "/${strReference}/${strRepoFile}" . ($bCompress ? qw{.} . COMPRESS_EXT : ''),
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strRepoFile}" . ($bCompress ? qw{.} . COMPRESS_EXT : ''),
+ {bHard => true});
}
# Else log the reference
else
@@ -379,7 +345,7 @@ sub processManifest
$iHostConfigIdx, $strQueueKey, $strRepoFile, OP_BACKUP_FILE,
[$strDbFile, $strRepoFile, $lSize,
$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, false),
- optionGet(OPTION_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, $bCompress,
+ optionGet(OPTION_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, $strBackupLabel, $bCompress,
$oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_TIMESTAMP, false),
$bIgnoreMissing, optionGet(OPTION_CHECKSUM_PAGE) && isChecksumPage($strRepoFile) ? $hStartLsnParam : undef]);
@@ -428,7 +394,7 @@ sub processManifest
# A keep-alive is required here because if there are a large number of resumed files that need to be checksummed
# then the remote might timeout while waiting for a command.
- $oProtocolMaster->keepAlive();
+ protocolKeepAlive();
}
# Validate the manifest
@@ -458,12 +424,7 @@ sub process
my $lTimestampStart = time();
# Initialize the local file object
- my $oFileLocal = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(NONE)
- );
+ my $oStorageRepo = storageRepo();
# Store local type, compress, and hardlink options since they can be modified by the process
my $strType = optionGet(OPTION_TYPE);
@@ -471,17 +432,11 @@ sub process
my $bHardLink = optionGet(OPTION_HARDLINK);
# Create the cluster backup and history path
- $oFileLocal->pathCreate(PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY, undef, true, true, optionGet(OPTION_REPO_SYNC));
+ $oStorageRepo->pathCreate(
+ STORAGE_REPO_BACKUP . qw(/) . PATH_BACKUP_HISTORY, {bIgnoreExists => true, bCreateParent => true});
# Load the backup.info
- my $oBackupInfo = new pgBackRest::Backup::Info($oFileLocal->pathGet(PATH_BACKUP_CLUSTER));
-
- # Build backup tmp and config
- my $strBackupTmpPath = $oFileLocal->pathGet(PATH_BACKUP_TMP);
- my $strBackupConfFile = $oFileLocal->pathGet(PATH_BACKUP_TMP, 'backup.manifest');
-
- # Declare the backup manifest
- my $oBackupManifest = new pgBackRest::Manifest($strBackupConfFile, false);
+ my $oBackupInfo = new pgBackRest::Backup::Info($oStorageRepo->pathGet(STORAGE_REPO_BACKUP));
# Initialize database objects
my $oDbMaster = undef;
@@ -506,12 +461,7 @@ sub process
}
# Initialize the master file object
- my $oFileMaster = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(DB, $self->{iMasterRemoteIdx})
- );
+ my $oStorageDbMaster = storageDb({iRemoteIdx => $self->{iMasterRemoteIdx}});
# Determine the database paths
my $strDbMasterPath = optionGet(optionIndex(OPTION_DB_PATH, $self->{iMasterRemoteIdx}));
@@ -534,7 +484,7 @@ sub process
if (defined($strBackupLastPath) && $oBackupInfo->confirmDb($strBackupLastPath, $strDbVersion, $ullDbSysId))
{
$oLastManifest = new pgBackRest::Manifest(
- $oFileLocal->pathGet(PATH_BACKUP_CLUSTER, "${strBackupLastPath}/" . FILE_MANIFEST));
+ $oStorageRepo->pathGet(STORAGE_REPO_BACKUP . "/${strBackupLastPath}/" . FILE_MANIFEST));
&log(INFO, 'last backup label = ' . $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) .
', version = ' . $oLastManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION));
@@ -566,6 +516,132 @@ sub process
}
}
+ # Search cluster directory for an aborted backup
+ my $strBackupLabel;
+ my $oAbortedManifest;
+ my $strBackupPath;
+
+ foreach my $strAbortedBackup ($oStorageRepo->list(
+ STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))
+ {
+ # Aborted backups have a copy of the manifest but no main
+ if ($oStorageRepo->exists(STORAGE_REPO_BACKUP . "/${strAbortedBackup}/" . FILE_MANIFEST_COPY) &&
+ !$oStorageRepo->exists(STORAGE_REPO_BACKUP . "/${strAbortedBackup}/" . FILE_MANIFEST))
+ {
+ my $bUsable;
+ my $strReason = "resume is disabled";
+ $strBackupPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP . "/${strAbortedBackup}");
+
+ # Attempt to read the manifest file in the aborted backup to see if it can be used. If any error at all occurs then the
+ # backup will be considered unusable and a resume will not be attempted.
+ if (optionGet(OPTION_RESUME))
+ {
+ $strReason = "unable to read ${strBackupPath}/" . FILE_MANIFEST;
+
+ eval
+ {
+ # Load the aborted manifest
+ $oAbortedManifest = new pgBackRest::Manifest("${strBackupPath}/" . FILE_MANIFEST);
+
+ # Key and values that do not match
+ my $strKey;
+ my $strValueNew;
+ my $strValueAborted;
+
+ # Check version
+ if ($oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION) ne BACKREST_VERSION)
+ {
+ $strKey = INI_KEY_VERSION;
+ $strValueNew = BACKREST_VERSION;
+ $strValueAborted = $oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION);
+ }
+ # Check format
+ elsif ($oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_FORMAT) ne BACKREST_FORMAT)
+ {
+ $strKey = INI_KEY_FORMAT;
+ $strValueNew = BACKREST_FORMAT;
+ $strValueAborted = $oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_FORMAT);
+ }
+ # Check backup type
+ elsif ($oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE) ne $strType)
+ {
+ $strKey = MANIFEST_KEY_TYPE;
+ $strValueNew = $strType;
+ $strValueAborted = $oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE);
+ }
+ # Check prior label
+ elsif ($oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, false, '') ne
+ (defined($strBackupLastPath) ? $strBackupLastPath : ''))
+ {
+ $strKey = MANIFEST_KEY_PRIOR;
+ $strValueNew = defined($strBackupLastPath) ? $strBackupLastPath : '';
+ $strValueAborted =
+ $oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, false, '');
+ }
+ # Check compression
+ elsif ($oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS) !=
+ optionGet(OPTION_COMPRESS))
+ {
+ $strKey = MANIFEST_KEY_COMPRESS;
+ $strValueNew = optionGet(OPTION_COMPRESS);
+ $strValueAborted = $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS);
+ }
+ # Check hardlink
+ elsif ($oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK) !=
+ optionGet(OPTION_HARDLINK))
+ {
+ $strKey = MANIFEST_KEY_HARDLINK;
+ $strValueNew = optionGet(OPTION_HARDLINK);
+ $strValueAborted = $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK);
+ }
+
+ # If key is defined then something didn't match
+ if (defined($strKey))
+ {
+ $strReason = "new ${strKey} '${strValueNew}' does not match aborted ${strKey} '${strValueAborted}'";
+ }
+ # Else the backup can be resumed
+ else
+ {
+ $bUsable = true;
+ }
+
+ return true;
+ }
+ or do
+ {
+ $bUsable = false;
+ }
+ }
+
+ # If the backup is usable then set the backup label
+ if ($bUsable)
+ {
+ $strBackupLabel = $strAbortedBackup;
+ }
+ else
+ {
+ &log(WARN, "aborted backup ${strAbortedBackup} cannot be resumed: ${strReason}");
+ &log(TEST, TEST_BACKUP_NORESUME);
+
+ $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strAbortedBackup}", {bRecurse => true});
+ undef($oAbortedManifest);
+ }
+
+ last;
+ }
+ }
+
+ # If backup label is not defined then create the label and path.
+ if (!defined($strBackupLabel))
+ {
+ $strBackupLabel = backupLabel($oStorageRepo, $strType, $strBackupLastPath, $lTimestampStart);
+ $strBackupPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP . "/${strBackupLabel}");
+ }
+
+ # Declare the backup manifest
+ my $oBackupManifest = new pgBackRest::Manifest("$strBackupPath/" . FILE_MANIFEST, false);
+
# Backup settings
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, $strType);
$oBackupManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, $lTimestampStart);
@@ -625,7 +701,7 @@ sub process
}
# Check if Postgres is running and if so only continue when forced
- if ($oFileMaster->exists(PATH_DB_ABSOLUTE, $strDbMasterPath . '/' . DB_FILE_POSTMASTERPID))
+ if ($oStorageDbMaster->exists($strDbMasterPath . '/' . DB_FILE_POSTMASTERPID))
{
if (optionGet(OPTION_FORCE))
{
@@ -712,140 +788,38 @@ sub process
$oBackupManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE, undef, optionGet(OPTION_CHECKSUM_PAGE));
# Build the manifest
- $oBackupManifest->build($oFileMaster, $strDbVersion, $strDbMasterPath, $oLastManifest, optionGet(OPTION_ONLINE),
+ $oBackupManifest->build($oStorageDbMaster, $strDbVersion, $strDbMasterPath, $oLastManifest, optionGet(OPTION_ONLINE),
$hTablespaceMap, $hDatabaseMap);
&log(TEST, TEST_MANIFEST_BUILD);
- # Check if an aborted backup exists for this stanza
- if (-e $strBackupTmpPath)
+ # If resuming from an aborted backup
+ if (defined($oAbortedManifest))
{
- my $bUsable;
- my $strReason = "resume is disabled";
- my $oAbortedManifest;
+ &log(WARN, "aborted backup ${strBackupLabel} of same type exists, will be cleaned to remove invalid files and resumed");
+ &log(TEST, TEST_BACKUP_RESUME);
- # Attempt to read the manifest file in the aborted backup to see if it can be used. If any error at all occurs then the
- # backup will be considered unusable and a resume will not be attempted.
- if (optionGet(OPTION_RESUME))
- {
- $strReason = "unable to read ${strBackupTmpPath}/backup.manifest";
-
- eval
- {
- # Load the aborted manifest
- $oAbortedManifest = new pgBackRest::Manifest("${strBackupTmpPath}/backup.manifest");
-
- # Key and values that do not match
- my $strKey;
- my $strValueNew;
- my $strValueAborted;
-
- # Check version
- if ($oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION) ne
- $oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION))
- {
- $strKey = INI_KEY_VERSION;
- $strValueNew = $oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION);
- $strValueAborted = $oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION);
- }
- # Check format
- elsif ($oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_FORMAT) ne
- $oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_FORMAT))
- {
- $strKey = INI_KEY_FORMAT;
- $strValueNew = $oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_FORMAT);
- $strValueAborted = $oAbortedManifest->get(INI_SECTION_BACKREST, INI_KEY_FORMAT);
- }
- # Check backup type
- elsif ($oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE) ne
- $oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE))
- {
- $strKey = MANIFEST_KEY_TYPE;
- $strValueNew = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE);
- $strValueAborted = $oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE);
- }
- # Check prior label
- elsif ($oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, false, '') ne
- $oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, false, ''))
- {
- $strKey = MANIFEST_KEY_PRIOR;
- $strValueNew = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, false, '');
- $strValueAborted = $oAbortedManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, false, '');
- }
- # Check compression
- elsif ($oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS) ne
- $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS))
- {
- $strKey = MANIFEST_KEY_COMPRESS;
- $strValueNew = $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS);
- $strValueAborted = $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS);
- }
- # Check hardlink
- elsif ($oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK) ne
- $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK))
- {
- $strKey = MANIFEST_KEY_HARDLINK;
- $strValueNew = $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK);
- $strValueAborted = $oAbortedManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK);
- }
-
- # If key is defined then something didn't match
- if (defined($strKey))
- {
- $strReason = "new ${strKey} '${strValueNew}' does not match aborted ${strKey} '${strValueAborted}'";
- }
- # Else the backup can be resumed
- else
- {
- $bUsable = true;
- }
-
- return true;
- }
- or do
- {
- $bUsable = false;
- }
- }
-
- # If the aborted backup is usable then clean it
- if ($bUsable)
- {
- &log(WARN, 'aborted backup of same type exists, will be cleaned to remove invalid files and resumed');
- &log(TEST, TEST_BACKUP_RESUME);
-
- # Clean the old backup tmp path
- $self->tmpClean($oFileLocal, $oBackupManifest, $oAbortedManifest);
- }
- # Else remove it
- else
- {
- &log(WARN, "aborted backup exists, but cannot be resumed (${strReason}) - will be dropped and recreated");
- &log(TEST, TEST_BACKUP_NORESUME);
-
- remove_tree($oFileLocal->pathGet(PATH_BACKUP_TMP))
- or confess &log(ERROR, "unable to delete tmp path: ${strBackupTmpPath}");
- $oFileLocal->pathCreate(PATH_BACKUP_TMP, undef, undef, false, true);
- }
+ # Clean the backup path before resuming
+ $self->resumeClean($oStorageRepo, $strBackupLabel, $oBackupManifest, $oAbortedManifest);
}
- # Else create the backup tmp path
+ # Else create the backup path
else
{
- logDebugMisc($strOperation, "create temp backup path ${strBackupTmpPath}");
- $oFileLocal->pathCreate(PATH_BACKUP_TMP, undef, undef, false, true);
+ logDebugMisc($strOperation, "create backup path ${strBackupPath}");
+ $oStorageRepo->pathCreate(STORAGE_REPO_BACKUP . "/${strBackupLabel}");
}
# Save the backup manifest
- $oBackupManifest->save();
+ $oBackupManifest->saveCopy();
# Perform the backup
my $lBackupSizeTotal =
$self->processManifest(
- $oFileMaster, $strDbMasterPath, $strDbCopyPath, $strType, $strDbVersion, $bCompress, $bHardLink, $oBackupManifest,
+ $strDbMasterPath, $strDbCopyPath, $strType, $strDbVersion, $bCompress, $bHardLink, $oBackupManifest, $strBackupLabel,
$strLsnStart);
&log(INFO, "${strType} backup size = " . fileSizeFormat($lBackupSizeTotal));
# Master file object no longer needed
- undef($oFileMaster);
+ undef($oStorageDbMaster);
# Stop backup (unless --no-online is set)
my $strArchiveStop = undef;
@@ -863,26 +837,26 @@ sub process
foreach my $strFile (sort(keys(%{$oFileHash})))
{
# Only save the file if it has content
- if (defined($$oFileHash{$strFile}))
+ if (defined($oFileHash->{$strFile}))
{
- my $strFileName = $oFileLocal->pathGet(PATH_BACKUP_TMP, $strFile);
+ my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];
- # Write content out to a file
- fileStringWrite($strFileName, $$oFileHash{$strFile}, false);
-
- # Compress if required
+ # Add compression filter
if ($bCompress)
{
- $oFileLocal->compress(PATH_BACKUP_ABSOLUTE, $strFileName);
- $strFileName .= '.' . $oFileLocal->{strCompressExtension};
+ push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP});
}
+ my $oDestinationFileIo = $oStorageRepo->openWrite(
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFile}" . ($bCompress ? qw{.} . COMPRESS_EXT : ''),
+ {rhyFilter => $rhyFilter});
+
+ # Write content out to a file
+ $oStorageRepo->put($oDestinationFileIo, $oFileHash->{$strFile});
+
# Add file to manifest
$oBackupManifest->fileAdd(
- $strFile,
- (fileStat($strFileName))->mtime,
- length($$oFileHash{$strFile}),
- $oFileLocal->hash(PATH_BACKUP_ABSOLUTE, $strFileName, $bCompress), true);
+ $strFile, time(), length($oFileHash->{$strFile}), $oDestinationFileIo->result(STORAGE_FILTER_SHA), true);
&log(DETAIL, "wrote '${strFile}' file returned from pg_stop_backup()");
}
@@ -900,20 +874,20 @@ sub process
# will be consistent - at least not here.
if (optionGet(OPTION_ONLINE) && optionGet(OPTION_BACKUP_ARCHIVE_CHECK))
{
- # Save the backup manifest a second time - before getting archive logs in case that fails
- $oBackupManifest->save();
+ # Save the backup manifest before getting archive logs in case of failure
+ $oBackupManifest->saveCopy();
# Create the modification time for the archive logs
my $lModificationTime = time();
# After the backup has been stopped, need to make a copy of the archive logs to make the db consistent
logDebugMisc($strOperation, "retrieve archive logs ${strArchiveStart}:${strArchiveStop}");
- my $strArchiveId = new pgBackRest::Archive::ArchiveGet()->getArchiveId($oFileLocal);
+ my $strArchiveId = new pgBackRest::Archive::ArchiveGet()->getArchiveId();
my @stryArchive = lsnFileRange($strLsnStart, $strLsnStop, $strDbVersion);
foreach my $strArchive (@stryArchive)
{
- my $strArchiveFile = walSegmentFind($oFileLocal, $strArchiveId, $strArchive, optionGet(OPTION_ARCHIVE_TIMEOUT));
+ my $strArchiveFile = walSegmentFind($oStorageRepo, $strArchiveId, $strArchive, optionGet(OPTION_ARCHIVE_TIMEOUT));
$strArchive = substr($strArchiveFile, 0, 24);
if (optionGet(OPTION_BACKUP_ARCHIVE_COPY))
@@ -921,82 +895,62 @@ sub process
logDebugMisc($strOperation, "archive: ${strArchive} (${strArchiveFile})");
# Copy the log file from the archive repo to the backup
- my $strDestinationFile = MANIFEST_TARGET_PGDATA . "/pg_xlog/${strArchive}" .
- ($bCompress ? ".$oFileLocal->{strCompressExtension}" : '');
- my $bArchiveCompressed = $strArchiveFile =~ "^.*\.$oFileLocal->{strCompressExtension}\$";
+ my $bArchiveCompressed = $strArchiveFile =~ ('^.*\.' . COMPRESS_EXT . '\$');
- my ($bCopyResult, $strCopyChecksum, $lCopySize) =
- $oFileLocal->copy(PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strArchiveFile}",
- PATH_BACKUP_TMP, $strDestinationFile,
- $bArchiveCompressed, $bCompress,
- undef, $lModificationTime, undef, true);
+ $oStorageRepo->copy(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strArchiveFile}",
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/" . MANIFEST_TARGET_PGDATA . "/pg_xlog/${strArchive}" .
+ ($bCompress ? qw{.} . COMPRESS_EXT : ''));
# Add the archive file to the manifest so it can be part of the restore and checked in validation
my $strPathLog = MANIFEST_TARGET_PGDATA . '/pg_xlog';
my $strFileLog = "${strPathLog}/${strArchive}";
- # Compare the checksum against the one already in the archive log name
- if ($strArchiveFile !~ "^${strArchive}-${strCopyChecksum}(\\.$oFileLocal->{strCompressExtension}){0,1}\$")
- {
- confess &log(ERROR, "error copying WAL segment '${strArchiveFile}' to backup - checksum recorded with " .
- "file does not match actual checksum of '${strCopyChecksum}'", ERROR_CHECKSUM);
- }
-
# Add file to manifest
- $oBackupManifest->fileAdd($strFileLog, $lModificationTime, $lCopySize, $strCopyChecksum, true);
+ $oBackupManifest->fileAdd(
+ $strFileLog, $lModificationTime, PG_WAL_SEGMENT_SIZE, substr($strArchiveFile, 25, 40), true);
}
}
}
- # Create label for new backup
- my $lTimestampStop = time();
- my $strBackupLabel = backupLabel($oFileLocal, $strType, $strBackupLastPath, $lTimestampStop);
-
# Record timestamp stop in the config
+ my $lTimestampStop = time();
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, $lTimestampStop + 0);
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel);
+ # Sync all paths in the backup cluster path
+ $oStorageRepo->pathSync(STORAGE_REPO_BACKUP . "/${strBackupLabel}", {bRecurse => true});
+
# Final save of the backup manifest
$oBackupManifest->save();
- # Sync all paths in the backup tmp path
- if (optionGet(OPTION_REPO_SYNC))
- {
- $oFileLocal->pathSync(PATH_BACKUP_TMP, undef, true);
- }
-
&log(INFO, "new backup label = ${strBackupLabel}");
- # Make a compressed copy of the manifest for history
- $oFileLocal->copy(PATH_BACKUP_TMP, FILE_MANIFEST,
- PATH_BACKUP_TMP, FILE_MANIFEST . '.gz',
- undef, true);
+ # Copy a compressed version of the manifest to history
+ $oStorageRepo->copy(
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/" . FILE_MANIFEST,
+ $oStorageRepo->openWrite(
+ STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . qw{/} . substr($strBackupLabel, 0, 4) .
+ "/${strBackupLabel}.manifest." . COMPRESS_EXT,
+ {rhyFilter => [{strClass => STORAGE_FILTER_GZIP}], bPathCreate => true, bAtomic => true}));
- # Move the backup tmp path to complete the backup
- logDebugMisc($strOperation, "move ${strBackupTmpPath} to " . $oFileLocal->pathGet(PATH_BACKUP_CLUSTER, $strBackupLabel));
- $oFileLocal->move(PATH_BACKUP_TMP, undef, PATH_BACKUP_CLUSTER, $strBackupLabel);
-
- # Copy manifest to history
- $oFileLocal->move(PATH_BACKUP_CLUSTER, "${strBackupLabel}/" . FILE_MANIFEST . '.gz',
- PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY . qw{/} . substr($strBackupLabel, 0, 4) .
- "/${strBackupLabel}.manifest.gz", true, optionGet(OPTION_REPO_SYNC));
+ # Sync history path
+ $oStorageRepo->pathSync(STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY);
# Create a link to the most recent backup
- $oFileLocal->remove(PATH_BACKUP_CLUSTER, LINK_LATEST);
+ $oStorageRepo->remove(STORAGE_REPO_BACKUP . qw(/) . LINK_LATEST);
- if (optionGet(OPTION_REPO_LINK))
+ if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK))
{
- $oFileLocal->linkCreate(PATH_BACKUP_CLUSTER, $strBackupLabel, PATH_BACKUP_CLUSTER, LINK_LATEST, undef, true);
+ $oStorageRepo->linkCreate(
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}", STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST, {bRelative => true});
}
# Save backup info
$oBackupInfo->add($oBackupManifest);
- # Sync the cluster path
- if (optionGet(OPTION_REPO_SYNC))
- {
- $oFileLocal->pathSync(PATH_BACKUP_CLUSTER);
- }
+ # Sync backup root path
+ $oStorageRepo->pathSync(STORAGE_REPO_BACKUP);
# Return from function and log return values if any
return logDebugReturn($strOperation);
diff --git a/lib/pgBackRest/Backup/Common.pm b/lib/pgBackRest/Backup/Common.pm
index eb0829a5b..df02db81e 100644
--- a/lib/pgBackRest/Backup/Common.pm
+++ b/lib/pgBackRest/Backup/Common.pm
@@ -15,8 +15,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
use pgBackRest::Manifest;
####################################################################################################################################
@@ -127,14 +127,14 @@ sub backupLabelFormat
$strOperation,
$strType,
$strBackupLabelLast,
- $lTimestampStop
+ $lTimestampStart
) =
logDebugParam
(
__PACKAGE__ . '::backupLabelFormat', \@_,
{name => 'strType', trace => true},
{name => 'strBackupLabelLast', required => false, trace => true},
- {name => 'lTimestampStop', trace => true}
+ {name => 'lTimestampTart', trace => true}
);
# Full backup label
@@ -149,7 +149,7 @@ sub backupLabelFormat
}
# Format the timestamp and add the full indicator
- $strBackupLabel = timestampFileFormat(undef, $lTimestampStop) . 'F';
+ $strBackupLabel = timestampFileFormat(undef, $lTimestampStart) . 'F';
}
# Else diff or incr label
else
@@ -164,7 +164,7 @@ sub backupLabelFormat
$strBackupLabel = substr($strBackupLabelLast, 0, 16);
# Format the timestamp
- $strBackupLabel .= '_' . timestampFileFormat(undef, $lTimestampStop);
+ $strBackupLabel .= '_' . timestampFileFormat(undef, $lTimestampStart);
# Add the diff indicator
if ($strType eq BACKUP_TYPE_DIFF)
@@ -199,36 +199,38 @@ sub backupLabel
my
(
$strOperation,
- $oFile,
+ $oStorageRepo,
$strType,
$strBackupLabelLast,
- $lTimestampStop
+ $lTimestampStart
) =
logDebugParam
(
__PACKAGE__ . '::backupLabelFormat', \@_,
- {name => 'oFile', trace => true},
+ {name => 'oStorageRepo', trace => true},
{name => 'strType', trace => true},
{name => 'strBackupLabelLast', required => false, trace => true},
- {name => 'lTimestampStop', trace => true}
+ {name => 'lTimestampStart', trace => true}
);
# Create backup label
- my $strBackupLabel = backupLabelFormat($strType, $strBackupLabelLast, $lTimestampStop);
+ my $strBackupLabel = backupLabelFormat($strType, $strBackupLabelLast, $lTimestampStart);
# Make sure that the timestamp has not already been used by a prior backup. This is unlikely for online backups since there is
# already a wait after the manifest is built but it's still possible if the remote and local systems don't have synchronized
# clocks. In practice this is most useful for making offline testing faster since it allows the wait after manifest build to
# be skipped by dealing with any backup label collisions here.
- if (fileList($oFile->pathGet(PATH_BACKUP_CLUSTER),
- {strExpression =>
- ($strType eq BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStop) .
- ($strType eq BACKUP_TYPE_FULL ? 'F' : '(D|I)$')}) ||
- fileList($oFile->pathGet(PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStop)),
- {strExpression =>
- ($strType eq BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStop) .
- ($strType eq BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.' . $oFile->{strCompressExtension}),
- bIgnoreMissing => true}))
+ if ($oStorageRepo->list(
+ STORAGE_REPO_BACKUP,
+ {strExpression =>
+ ($strType eq BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) .
+ ($strType eq BACKUP_TYPE_FULL ? 'F' : '(D|I)$')}) ||
+ $oStorageRepo->list(
+ STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStart),
+ {strExpression =>
+ ($strType eq BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) .
+ ($strType eq BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.' . COMPRESS_EXT . qw{$}),
+ bIgnoreMissing => true}))
{
waitRemainder();
$strBackupLabel = backupLabelFormat($strType, $strBackupLabelLast, time());
diff --git a/lib/pgBackRest/Backup/File.pm b/lib/pgBackRest/Backup/File.pm
index 6042d3ee1..335e15c90 100644
--- a/lib/pgBackRest/Backup/File.pm
+++ b/lib/pgBackRest/Backup/File.pm
@@ -12,14 +12,18 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use Storable qw(dclone);
-use pgBackRest::DbVersion;
+use pgBackRest::Backup::Filter::PageChecksum;
use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
+use pgBackRest::DbVersion;
use pgBackRest::Manifest;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Filter::Gzip;
+use pgBackRest::Storage::Filter::Sha;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# Result constants
@@ -33,126 +37,6 @@ use constant BACKUP_FILE_RECOPY => 2;
use constant BACKUP_FILE_SKIP => 3;
push @EXPORT, qw(BACKUP_FILE_SKIP);
-####################################################################################################################################
-# Load the C library if present
-####################################################################################################################################
-my $bLibC = false;
-
-eval
-{
- # Load the C library only if page checksums are required
- require pgBackRest::LibC;
- pgBackRest::LibC->import(qw(:checksum));
-
- $bLibC = true;
-
- return 1;
-} or do {};
-
-####################################################################################################################################
-# isLibC
-#
-# Does the C library exist?
-####################################################################################################################################
-sub isLibC
-{
- return $bLibC;
-}
-
-push @EXPORT, qw(isLibC);
-
-####################################################################################################################################
-# backupChecksumPage
-####################################################################################################################################
-sub backupChecksumPage
-{
- my $rExtraParam = shift;
- my $tBufferRef = shift;
- my $iBufferSize = shift;
- my $iBufferOffset = shift;
- my $hExtra = shift;
-
- # Initialize the extra hash
- if (!defined($hExtra->{bValid}))
- {
- $hExtra->{bValid} = true;
- }
-
- # Return when buffer is 0
- if ($iBufferSize == 0)
- {
- # Make sure valid is set for 0 length files
- if ($iBufferOffset == 0 && !defined($hExtra->{bValid}))
- {
- $hExtra->{bValid} = true;
- }
-
- return;
- }
-
- # Error if offset is not divisible by page size
- if ($iBufferOffset % PG_PAGE_SIZE != 0)
- {
- confess &log(ASSERT, "should not be possible to see misaligned buffer offset ${iBufferOffset}, buffer size ${iBufferSize}");
- }
-
- # If the buffer is not divisible by 0 then it's not valid
- if ($iBufferSize % PG_PAGE_SIZE != 0)
- {
- if (defined($hExtra->{bAlign}))
- {
- confess &log(ASSERT, "should not be possible to see two misaligned blocks in a row");
- }
-
- $hExtra->{bValid} = false;
- $hExtra->{bAlign} = false;
- delete($hExtra->{iyPageError});
- }
- elsif ($iBufferSize > 0)
- {
- # Calculate offset to the first block in the buffer
- my $iBlockOffset = int($iBufferOffset / PG_PAGE_SIZE) + ($rExtraParam->{iSegmentNo} * 131072);
-
- if (!pageChecksumBufferTest(
- $$tBufferRef, $iBufferSize, $iBlockOffset, PG_PAGE_SIZE, $rExtraParam->{iWalId},
- $rExtraParam->{iWalOffset}))
- {
- $hExtra->{bValid} = false;
-
- # Now figure out exactly where the errors occurred. It would be more efficient if the checksum function returned an
- # array, but we're hoping there won't be that many errors to scan so this should work fine.
- for (my $iBlockNo = 0; $iBlockNo < int($iBufferSize / PG_PAGE_SIZE); $iBlockNo++)
- {
- my $iBlockNoStart = $iBlockOffset + $iBlockNo;
-
- if (!pageChecksumTest(
- substr($$tBufferRef, $iBlockNo * PG_PAGE_SIZE, PG_PAGE_SIZE), $iBlockNoStart, PG_PAGE_SIZE,
- $rExtraParam->{iWalId}, $rExtraParam->{iWalOffset}))
- {
- my $iLastIdx = defined($hExtra->{iyPageError}) ? @{$hExtra->{iyPageError}} - 1 : 0;
- my $iyLast = defined($hExtra->{iyPageError}) ? $hExtra->{iyPageError}[$iLastIdx] : undef;
-
- if (!defined($iyLast) || (!ref($iyLast) && $iyLast != $iBlockNoStart - 1) ||
- (ref($iyLast) && $iyLast->[1] != $iBlockNoStart - 1))
- {
- push(@{$hExtra->{iyPageError}}, $iBlockNoStart);
- }
- elsif (!ref($iyLast))
- {
- $hExtra->{iyPageError}[$iLastIdx] = undef;
- push(@{$hExtra->{iyPageError}[$iLastIdx]}, $iyLast);
- push(@{$hExtra->{iyPageError}[$iLastIdx]}, $iBlockNoStart);
- }
- else
- {
- $hExtra->{iyPageError}[$iLastIdx][1] = $iBlockNoStart;
- }
- }
- }
- }
- }
-}
-
####################################################################################################################################
# backupFile
####################################################################################################################################
@@ -162,12 +46,12 @@ sub backupFile
my
(
$strOperation,
- $oFile, # File object
$strDbFile, # Database file to backup
$strRepoFile, # Location in the repository to copy to
$lSizeFile, # File size
$strChecksum, # File checksum to be checked
$bChecksumPage, # Should page checksums be calculated?
+ $strBackupLabel, # Label of current backup
$bDestinationCompress, # Compress destination file
$lModificationTime, # File modification time
$bIgnoreMissing, # Is it OK if the file is missing?
@@ -176,73 +60,93 @@ sub backupFile
logDebugParam
(
__PACKAGE__ . '::backupFile', \@_,
- {name => 'oFile', trace => true},
{name => 'strDbFile', trace => true},
{name => 'strRepoFile', trace => true},
{name => 'lSizeFile', trace => true},
{name => 'strChecksum', required => false, trace => true},
{name => 'bChecksumPage', trace => true},
+ {name => 'strBackupLabel', trace => true},
{name => 'bDestinationCompress', trace => true},
{name => 'lModificationTime', trace => true},
{name => 'bIgnoreMissing', default => true, trace => true},
{name => 'hExtraParam', required => false, trace => true},
);
+ my $oStorageRepo = storageRepo(); # Repo storage
my $iCopyResult = BACKUP_FILE_COPY; # Copy result
my $strCopyChecksum; # Copy checksum
my $rExtra; # Page checksum result
my $lCopySize; # Copy Size
my $lRepoSize; # Repo size
+ # Add compression suffix if needed
+ my $strFileOp = $strRepoFile . ($bDestinationCompress ? '.' . COMPRESS_EXT : '');
+
# If checksum is defined then the file already exists but needs to be checked
my $bCopy = true;
- # Add compression suffix if needed
- my $strFileOp = $strRepoFile . ($bDestinationCompress ? '.' . $oFile->{strCompressExtension} : '');
-
if (defined($strChecksum))
{
- ($strCopyChecksum, $lCopySize) =
- $oFile->hashSize(PATH_BACKUP_TMP, $strFileOp, $bDestinationCompress);
+ # Get the checksum
+ ($strCopyChecksum, $lCopySize) = $oStorageRepo->hashSize(
+ $oStorageRepo->openRead(
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}",
+ {rhyFilter => $bDestinationCompress ?
+ [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
+ # Determine if the file needs to be recopied
$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);
- if ($bCopy)
- {
- $iCopyResult = BACKUP_FILE_RECOPY;
- }
- else
- {
- $iCopyResult = BACKUP_FILE_CHECKSUM;
- }
+ # Set copy result
+ $iCopyResult = $bCopy ? BACKUP_FILE_RECOPY : BACKUP_FILE_CHECKSUM;
}
+ # Copy the file
if ($bCopy)
{
- # Determine which segment no this is by checking for a numeric extension. No extension means segment 0.
+ # Add sha filter
+ my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];
+
+ # Add page checksum filter
if ($bChecksumPage)
{
- $hExtraParam->{iSegmentNo} = ($strDbFile =~ /\.[0-9]+$/) ? substr(($strDbFile =~ m/\.[0-9]+$/g)[0], 1) + 0 : 0;
+ # Determine which segment no this is by checking for a numeric extension. No extension means segment 0.
+ my $iSegmentNo = ($strDbFile =~ /\.[0-9]+$/) ? substr(($strDbFile =~ m/\.[0-9]+$/g)[0], 1) + 0 : 0;
+
+ push(
+ @{$rhyFilter},
+ {strClass => BACKUP_FILTER_PAGECHECKSUM,
+ rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]});
+ };
+
+ # Add compression
+ if ($bDestinationCompress)
+ {
+ push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP});
}
- # Copy the file from the database to the backup (will return false if the source file is missing)
- (my $bCopyResult, $strCopyChecksum, $lCopySize, $rExtra) = $oFile->copy(
- PATH_DB_ABSOLUTE, $strDbFile,
- PATH_BACKUP_TMP, $strFileOp,
- false, # Source is not compressed since it is the db directory
- $bDestinationCompress, # Destination should be compressed based on backup settings
- $bIgnoreMissing, # Ignore missing files
- undef, # Do not set modification time
- undef, # Do not set original mode
- true, # Create the destination directory if it does not exist
- undef, undef, undef, undef, # Unused
- $bChecksumPage ? # Function to process page checksums
- 'pgBackRest::Backup::File::backupChecksumPage' : undef,
- $hExtraParam, # Start LSN to pass to extra function
- false); # Don't copy via a temp file
+ # Open the file
+ my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => true});
- # If source file is missing then assume the database removed it (else corruption and nothing we can do!)
- if (!$bCopyResult)
+ # If source file exists
+ if (defined($oSourceFileIo))
+ {
+ # Copy the file
+ $oStorageRepo->copy(
+ $oSourceFileIo,
+ $oStorageRepo->openWrite(
+ STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}",
+ {bPathCreate => true, bProtocolCompress => !$bDestinationCompress}));
+
+ # Get sha checksum and size
+ $strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA);
+ $lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE);
+
+ # Get results of page checksum validation
+ $rExtra = $bChecksumPage ? $oSourceFileIo->result(BACKUP_FILTER_PAGECHECKSUM) : undef;
+ }
+ # Else if source file is missing the database removed it
+ else
{
$iCopyResult = BACKUP_FILE_SKIP;
}
@@ -252,7 +156,7 @@ sub backupFile
# compression may affect the actual repo size and this cannot be calculated in stream.
if ($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY || $iCopyResult == BACKUP_FILE_CHECKSUM)
{
- $lRepoSize = (fileStat($oFile->pathGet(PATH_BACKUP_TMP, $strFileOp)))->size;
+ $lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->size();
}
# Return from function and log return values if any
@@ -448,7 +352,8 @@ sub backupManifestUpdate
if ($lManifestSaveCurrent >= $lManifestSaveSize)
{
- $oManifest->save();
+ $oManifest->saveCopy();
+
logDebugMisc
(
$strOperation, 'save manifest',
diff --git a/lib/pgBackRest/Backup/Filter/PageChecksum.pm b/lib/pgBackRest/Backup/Filter/PageChecksum.pm
new file mode 100644
index 000000000..166f97853
--- /dev/null
+++ b/lib/pgBackRest/Backup/Filter/PageChecksum.pm
@@ -0,0 +1,196 @@
+####################################################################################################################################
+# Backup Page Checksum Filter
+####################################################################################################################################
+package pgBackRest::Backup::Filter::PageChecksum;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::DbVersion qw(PG_PAGE_SIZE);
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant BACKUP_FILTER_PAGECHECKSUM => __PACKAGE__;
+ push @EXPORT, qw(BACKUP_FILTER_PAGECHECKSUM);
+
+####################################################################################################################################
+# Load the C library if present
+####################################################################################################################################
+my $bLibC = false;
+
+eval
+{
+ # Load the C library only if page checksums are required
+ require pgBackRest::LibC;
+ pgBackRest::LibC->import(qw(:checksum));
+
+ $bLibC = true;
+
+ return 1;
+} or do {};
+
+####################################################################################################################################
+# isLibC
+#
+# Does the C library exist?
+####################################################################################################################################
+sub isLibC
+{
+ return $bLibC;
+}
+
+push @EXPORT, qw(isLibC);
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $self,
+ $iSegmentNo,
+ $iWalId,
+ $iWalOffset,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'self', trace => true},
+ {name => 'iSegmentNo', trace => true},
+ {name => 'iWalId', trace => true},
+ {name => 'iWalOffset', trace => true},
+ );
+
+ # Bless with new class
+ @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+ bless $self, $class;
+
+ # Set variables
+ $self->{iSegmentNo} = $iSegmentNo;
+ $self->{iWalId} = $iWalId;
+ $self->{iWalOffset} = $iWalOffset;
+
+ # Create the result object
+ $self->{hResult}{bValid} = true;
+ $self->{hResult}{bAlign} = true;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# read - validate page checksums
+####################################################################################################################################
+sub read
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+ my $iSize = shift;
+
+ # Call the io method
+ my $iActualSize = $self->SUPER::read($rtBuffer, $iSize);
+
+ # Validate page checksums for the read block
+ if ($iActualSize > 0)
+ {
+ # If the buffer is not divisible by 0 then it's not valid
+ if (!$self->{hResult}{bAlign} || ($iActualSize % PG_PAGE_SIZE != 0))
+ {
+ if (!$self->{hResult}{bAlign})
+ {
+ confess &log(ASSERT, "should not be possible to see two misaligned blocks in a row");
+ }
+
+ $self->{hResult}{bValid} = false;
+ $self->{hResult}{bAlign} = false;
+ delete($self->{hResult}{iyPageError});
+ }
+ else
+ {
+ # Calculate offset to the first block in the buffer
+ my $iBlockOffset = int(($self->size() - $iActualSize) / PG_PAGE_SIZE) + ($self->{iSegmentNo} * 131072);
+
+ if (!pageChecksumBufferTest(
+ $$rtBuffer, $iActualSize, $iBlockOffset, PG_PAGE_SIZE, $self->{iWalId},
+ $self->{iWalOffset}))
+ {
+ $self->{hResult}{bValid} = false;
+
+ # Now figure out exactly where the errors occurred. It would be more efficient if the checksum function returned an
+ # array, but we're hoping there won't be that many errors to scan so this should work fine.
+ for (my $iBlockNo = 0; $iBlockNo < int($iActualSize / PG_PAGE_SIZE); $iBlockNo++)
+ {
+ my $iBlockNoStart = $iBlockOffset + $iBlockNo;
+
+ if (!pageChecksumTest(
+ substr($$rtBuffer, $iBlockNo * PG_PAGE_SIZE, PG_PAGE_SIZE), $iBlockNoStart, PG_PAGE_SIZE,
+ $self->{iWalId}, $self->{iWalOffset}))
+ {
+ my $iLastIdx = defined($self->{hResult}{iyPageError}) ? @{$self->{hResult}{iyPageError}} - 1 : 0;
+ my $iyLast = defined($self->{hResult}{iyPageError}) ? $self->{hResult}{iyPageError}[$iLastIdx] : undef;
+
+ if (!defined($iyLast) || (!ref($iyLast) && $iyLast != $iBlockNoStart - 1) ||
+ (ref($iyLast) && $iyLast->[1] != $iBlockNoStart - 1))
+ {
+ push(@{$self->{hResult}{iyPageError}}, $iBlockNoStart);
+ }
+ elsif (!ref($iyLast))
+ {
+ $self->{hResult}{iyPageError}[$iLastIdx] = undef;
+ push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iyLast);
+ push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iBlockNoStart);
+ }
+ else
+ {
+ $self->{hResult}{iyPageError}[$iLastIdx][1] = $iBlockNoStart;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ # Return the actual size read
+ return $iActualSize;
+}
+
+####################################################################################################################################
+# close - close and set the result
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ if (defined($self->{hResult}))
+ {
+ # Set result
+ $self->resultSet(BACKUP_FILTER_PAGECHECKSUM, $self->{hResult});
+
+ # Delete the sha object
+ undef($self->{hResult});
+
+ # Close io
+ return $self->SUPER::close();
+ }
+}
+
+1;
diff --git a/lib/pgBackRest/Backup/Info.pm b/lib/pgBackRest/Backup/Info.pm
index 36a8930d7..da235e153 100644
--- a/lib/pgBackRest/Backup/Info.pm
+++ b/lib/pgBackRest/Backup/Info.pm
@@ -20,12 +20,11 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# File/path constants
@@ -115,6 +114,9 @@ sub new
$strBackupClusterPath,
$bValidate,
$bRequired,
+ $oStorage,
+ $bLoad, # Should the file attemp to be loaded?
+ $bIgnoreMissing, # Don't error on missing files
) =
logDebugParam
(
@@ -122,24 +124,50 @@ sub new
{name => 'strBackupClusterPath'},
{name => 'bValidate', default => true},
{name => 'bRequired', default => true},
+ {name => 'oStorage', optional => true, default => storageRepo()},
+ {name => 'bLoad', optional => true, default => true},
+ {name => 'bIgnoreMissing', optional => true, default => false},
);
# Build the backup info path/file name
my $strBackupInfoFile = "${strBackupClusterPath}/" . FILE_BACKUP_INFO;
- my $bExists = fileExists($strBackupInfoFile);
-
- # If the backup info file does not exist and is required, then throw an error
- # The backup.info is only allowed not to exist when running a stanza-create on a new install
- if (!$bExists && $bRequired)
- {
- confess &log(ERROR, "${strBackupClusterPath}/$strBackupInfoMissingMsg", ERROR_FILE_MISSING);
- }
+ my $self = {};
+ my $iResult = 0;
+ my $strResultMessage;
# Init object and store variables
- my $self = $class->SUPER::new($strBackupInfoFile, {bLoad => $bExists});
+ eval
+ {
+ $self = $class->SUPER::new($strBackupInfoFile, {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing,
+ oStorage => $oStorage});
+ return true;
+ }
+ or do
+ {
+ # Capture error information
+ $iResult = exceptionCode($EVAL_ERROR);
+ $strResultMessage = exceptionMessage($EVAL_ERROR->message());
+ };
+
+ if ($iResult != 0)
+ {
+ # If the backup info file does not exist and is required, then throw an error
+ # The backup info is only allowed not to exist when running a stanza-create on a new install
+ if ($iResult == ERROR_FILE_MISSING)
+ {
+ if ($bRequired)
+ {
+ confess &log(ERROR, "${strBackupClusterPath}/$strBackupInfoMissingMsg", ERROR_FILE_MISSING);
+ }
+ }
+ else
+ {
+ confess &log(ERROR, $strResultMessage, $iResult);
+ }
+ }
- $self->{bExists} = $bExists;
$self->{strBackupClusterPath} = $strBackupClusterPath;
+ $self->{oStorage} = $oStorage;
# Validate the backup info
if ($bValidate)
@@ -209,14 +237,15 @@ sub reconstruct
);
# Check for backups that are not in FILE_BACKUP_INFO
- foreach my $strBackup (fileList($self->{strBackupClusterPath}, {strExpression => backupRegExpGet(true, true, true)}))
+ foreach my $strBackup ($self->{oStorage}->list(
+ $self->{strBackupClusterPath}, {strExpression => backupRegExpGet(true, true, true)}))
{
my $strManifestFile = "$self->{strBackupClusterPath}/${strBackup}/" . FILE_MANIFEST;
# ??? Check for and move history files that were not moved before and maybe don't consider it to be an error when they
# can't be moved. This would also be true for the first move attempt in Backup->process();
- if (!$self->current($strBackup) && fileExists($strManifestFile))
+ if (!$self->current($strBackup) && $self->{oStorage}->exists($strManifestFile))
{
my $oManifest = pgBackRest::Manifest->new($strManifestFile);
@@ -298,12 +327,12 @@ sub reconstruct
my $strManifestFile = "$self->{strBackupClusterPath}/${strBackup}/" . FILE_MANIFEST;
my $strBackupPath = "$self->{strBackupClusterPath}/${strBackup}";
- if (!fileExists($strBackupPath))
+ if (!$self->{oStorage}->pathExists($strBackupPath))
{
&log(WARN, "backup ${strBackup} missing in repository removed from " . FILE_BACKUP_INFO);
$self->delete($strBackup);
}
- elsif (!fileExists($strManifestFile))
+ elsif (!$self->{oStorage}->exists($strManifestFile))
{
&log(WARN, "backup ${strBackup} missing manifest removed from " . FILE_BACKUP_INFO);
$self->delete($strBackup);
diff --git a/lib/pgBackRest/Check/Check.pm b/lib/pgBackRest/Check/Check.pm
index 16715f9ca..91e2da884 100644
--- a/lib/pgBackRest/Check/Check.pm
+++ b/lib/pgBackRest/Check/Check.pm
@@ -16,9 +16,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Db;
-use pgBackRest::File;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
####################################################################################################################################
# constructor
@@ -56,14 +55,6 @@ sub process
# Assign function parameters, defaults, and log debug info
my $strOperation = logDebugParam(__PACKAGE__ . '->process');
- # Initialize default file object
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(isRepoLocal() ? DB : BACKUP)
- );
-
# Initialize the database object
my $oDb = dbMasterGet();
@@ -88,7 +79,7 @@ sub process
eval
{
# Check that the backup info file is written and is valid for the current database of the stanza
- $self->backupInfoCheck($oFile);
+ $self->backupInfoCheck();
return true;
}
# If there is an unhandled error then confess
@@ -105,7 +96,7 @@ sub process
eval
{
# Check that the archive info file is written and is valid for the current database of the stanza
- ($strArchiveId) = new pgBackRest::Archive::ArchiveGet()->getCheck($oFile);
+ ($strArchiveId) = new pgBackRest::Archive::ArchiveGet()->getCheck();
return true;
}
or do
@@ -123,7 +114,7 @@ sub process
eval
{
- $strArchiveFile = walSegmentFind($oFile, $strArchiveId, $strWalSegment, $iArchiveTimeout);
+ $strArchiveFile = walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, $iArchiveTimeout);
return true;
}
# If this is a backrest error then capture the code and message else confess
@@ -146,7 +137,7 @@ sub process
{
&log(INFO,
"WAL segment ${strWalSegment} successfully stored in the archive at '" .
- $oFile->pathGet(PATH_BACKUP_ARCHIVE, "$strArchiveId/${strArchiveFile}") . "'");
+ storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/$strArchiveId/${strArchiveFile}") . "'");
}
else
{
@@ -163,7 +154,7 @@ sub process
&log(WARN,
"WAL segment ${strWalSegment} did not reach the archive:" . (defined($strArchiveId) ? $strArchiveId : '') . "\n" .
"HINT: Check the archive_command to ensure that all options are correct (especialy --stanza).\n" .
- "HINT: Check the PostreSQL server log for errors.");
+ "HINT: Check the PostgreSQL server log for errors.");
}
}
@@ -173,7 +164,6 @@ sub process
$strOperation,
{name => 'iResult', value => $iResult, trace => true}
);
-
}
####################################################################################################################################
@@ -189,16 +179,14 @@ sub backupInfoCheck
my
(
$strOperation,
- $oFile,
$strDbVersion,
$iControlVersion,
$iCatalogVersion,
- $ullDbSysId
+ $ullDbSysId,
) =
logDebugParam
(
__PACKAGE__ . '->backupInfoCheck', \@_,
- {name => 'oFile'},
{name => 'strDbVersion', required => false},
{name => 'iControlVersion', required => false},
{name => 'iCatalogVersion', required => false},
@@ -214,14 +202,14 @@ sub backupInfoCheck
($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId) = dbMasterGet()->info();
}
- if ($oFile->isRemote(PATH_BACKUP))
+ if (!isRepoLocal())
{
- $iDbHistoryId = $oFile->{oProtocol}->cmdExecute(
+ $iDbHistoryId = protocolGet(BACKUP)->cmdExecute(
OP_CHECK_BACKUP_INFO_CHECK, [$strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId]);
}
else
{
- $iDbHistoryId = (new pgBackRest::Backup::Info($oFile->pathGet(PATH_BACKUP_CLUSTER)))->check(
+ $iDbHistoryId = (new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP)))->check(
$strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId);
}
diff --git a/lib/pgBackRest/Common/Exception.pm b/lib/pgBackRest/Common/Exception.pm
index 6980f88f6..789fe3308 100644
--- a/lib/pgBackRest/Common/Exception.pm
+++ b/lib/pgBackRest/Common/Exception.pm
@@ -152,6 +152,10 @@ use constant ERROR_OPTION_COMMAND => ERROR_MIN
push @EXPORT, qw(ERROR_OPTION_COMMAND);
use constant ERROR_GROUP_MISSING => ERROR_MINIMUM + 66;
push @EXPORT, qw(ERROR_GROUP_MISSING);
+use constant ERROR_PATH_EXISTS => ERROR_MINIMUM + 67;
+ push @EXPORT, qw(ERROR_PATH_EXISTS);
+use constant ERROR_FILE_EXISTS => ERROR_MINIMUM + 68;
+ push @EXPORT, qw(ERROR_FILE_EXISTS);
use constant ERROR_INVALID_VALUE => ERROR_MAXIMUM - 2;
push @EXPORT, qw(ERROR_INVALID_VALUE);
diff --git a/lib/pgBackRest/Common/Ini.pm b/lib/pgBackRest/Common/Ini.pm
index 8e3f27e6f..a1e542352 100644
--- a/lib/pgBackRest/Common/Ini.pm
+++ b/lib/pgBackRest/Common/Ini.pm
@@ -8,6 +8,7 @@ use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
+use Digest::SHA;
use Exporter qw(import);
our @EXPORT = qw();
use Fcntl qw(:mode O_WRONLY O_CREAT O_TRUNC);
@@ -19,7 +20,6 @@ use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
-use pgBackRest::FileCommon;
use pgBackRest::Version;
####################################################################################################################################
@@ -43,6 +43,12 @@ use constant INI_KEY_FORMAT => 'backrest
use constant INI_KEY_VERSION => 'backrest-version';
push @EXPORT, qw(INI_KEY_VERSION);
+####################################################################################################################################
+# Ini file copy extension
+####################################################################################################################################
+use constant INI_COPY_EXT => '.copy';
+ push @EXPORT, qw(INI_COPY_EXT);
+
####################################################################################################################################
# Ini sort orders
####################################################################################################################################
@@ -64,14 +70,20 @@ sub new
my $self = {};
bless $self, $class;
+ # Load Storage::Helper module
+ require pgBackRest::Storage::Helper;
+ pgBackRest::Storage::Helper->import();
+
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strFileName},
my $bLoad,
my $strContent,
+ $self->{oStorage},
$self->{iInitFormat},
$self->{strInitVersion},
+ my $bIgnoreMissing,
) =
logDebugParam
(
@@ -79,13 +91,12 @@ sub new
{name => 'strFileName', trace => true},
{name => 'bLoad', optional => true, default => true, trace => true},
{name => 'strContent', optional => true, trace => true},
+ {name => 'oStorage', optional => true, default => storageLocal(), trace => true},
{name => 'iInitFormat', optional => true, default => BACKREST_FORMAT, trace => true},
{name => 'strInitVersion', optional => true, default => BACKREST_VERSION, trace => true},
+ {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
);
- # Set variables
- $self->{oContent} = {};
-
# Set changed to false
$self->{bModified} = false;
@@ -95,7 +106,7 @@ sub new
# Load the file if requested
if ($bLoad)
{
- $self->load();
+ $self->load($bIgnoreMissing);
}
# Load from a string if provided
elsif (defined($strContent))
@@ -103,8 +114,9 @@ sub new
$self->{oContent} = iniParse($strContent);
$self->headerCheck();
}
- # Else initialize
- else
+
+ # Initialize if not loading from string and file does not exist
+ if (!$self->{bExists} && !defined($strContent))
{
$self->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, $self->{iInitFormat});
$self->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, $self->{strInitVersion});
@@ -114,14 +126,59 @@ sub new
}
####################################################################################################################################
-# load() - load the ini.
+# loadVersion() - load a version (main or copy) of the ini file
+####################################################################################################################################
+sub loadVersion
+{
+ my $self = shift;
+ my $bCopy = shift;
+ my $bIgnoreError = shift;
+
+ # Load main
+ my $rstrContent = $self->{oStorage}->get(
+ $self->{oStorage}->openRead($self->{strFileName} . ($bCopy ? INI_COPY_EXT : ''), {bIgnoreMissing => $bIgnoreError}));
+
+ # If the file exists then attempt to parse it
+ if (defined($rstrContent))
+ {
+
+ my $rhContent = iniParse($$rstrContent, {bIgnoreInvalid => $bIgnoreError});
+
+ # If the content is valid then check the header
+ if (defined($rhContent))
+ {
+ $self->{oContent} = $rhContent;
+
+ # If the header is invalid then undef content
+ if (!$self->headerCheck({bIgnoreInvalid => $bIgnoreError}))
+ {
+ delete($self->{oContent});
+ }
+ }
+ }
+
+ return defined($self->{oContent});
+}
+
+####################################################################################################################################
+# load() - load the ini
####################################################################################################################################
sub load
{
my $self = shift;
+ my $bIgnoreMissing = shift;
+
+ # If main was not loaded then try the copy
+ if (!$self->loadVersion(false, true))
+ {
+ if (!$self->loadVersion(true, true))
+ {
+ return if $bIgnoreMissing;
+
+ confess &log(ERROR, "unable to open $self->{strFileName} or $self->{strFileName}" . INI_COPY_EXT, ERROR_FILE_MISSING);
+ }
+ }
- $self->{oContent} = iniParse(fileStringRead($self->{strFileName}));
- $self->headerCheck();
$self->{bExists} = true;
}
@@ -149,6 +206,7 @@ sub headerCheck
eval
{
+
# Make sure the ini is valid by testing checksum
my $strChecksum = $self->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, false);
my $strTestChecksum = $self->hash();
@@ -216,7 +274,7 @@ sub iniParse
logDebugParam
(
__PACKAGE__ . '::iniParse', \@_,
- {name => 'strContent', trace => true},
+ {name => 'strContent', required => false, trace => true},
{name => 'bRelaxed', optional => true, default => false, trace => true},
{name => 'bIgnoreInvalid', optional => true, default => false, trace => true},
);
@@ -232,7 +290,7 @@ sub iniParse
eval
{
# Read the INI file
- foreach my $strLine (split("\n", $strContent))
+ foreach my $strLine (split("\n", defined($strContent) ? $strContent : ''))
{
$strLine = trim($strLine);
@@ -323,13 +381,17 @@ sub save
{
my $self = shift;
+ # Save only if modified
if ($self->{bModified})
{
# Calculate the hash
$self->hash();
# Save the file
- fileStringWrite($self->{strFileName}, iniRender($self->{oContent}));
+ $self->{oStorage}->put($self->{strFileName}, iniRender($self->{oContent}));
+ $self->{oStorage}->pathSync(dirname($self->{strFileName}));
+ $self->{oStorage}->put($self->{strFileName} . INI_COPY_EXT, iniRender($self->{oContent}));
+ $self->{oStorage}->pathSync(dirname($self->{strFileName}));
$self->{bModified} = false;
# Indicate the file now exists
@@ -343,6 +405,22 @@ sub save
return false;
}
+####################################################################################################################################
+# saveCopy - save only a copy of the file.
+####################################################################################################################################
+sub saveCopy
+{
+ my $self = shift;
+
+ if ($self->{oStorage}->exists($self->{strFileName}))
+ {
+ confess &log(ASSERT, "cannot save copy only when '$self->{strFileName}' exists");
+ }
+
+ $self->hash();
+ $self->{oStorage}->put($self->{strFileName} . INI_COPY_EXT, iniRender($self->{oContent}));
+}
+
####################################################################################################################################
# iniRender() - render hash to standard INI format.
####################################################################################################################################
@@ -435,7 +513,6 @@ sub hash
delete($self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
# Calculate the checksum
- my $oChecksumContent = dclone($self->{oContent});
my $oSHA = Digest::SHA->new('sha1');
my $oJSON = JSON::PP->new()->canonical()->allow_nonref();
$oSHA->add($oJSON->encode($self->{oContent}));
diff --git a/lib/pgBackRest/Common/Io/Base.pm b/lib/pgBackRest/Common/Io/Base.pm
new file mode 100644
index 000000000..5a3210edb
--- /dev/null
+++ b/lib/pgBackRest/Common/Io/Base.pm
@@ -0,0 +1,126 @@
+####################################################################################################################################
+# Base IO/Filter Module
+####################################################################################################################################
+package pgBackRest::Common::Io::Base;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use Scalar::Util qw(blessed);
+
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant COMMON_IO_BASE => __PACKAGE__;
+ push @EXPORT, qw(COMMON_IO_BASE);
+
+####################################################################################################################################
+# Default buffer max
+####################################################################################################################################
+use constant COMMON_IO_BUFFER_MAX => 4194304;
+ push @EXPORT, qw(COMMON_IO_BUFFER_MAX);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Create the class hash
+ my $self = {};
+ bless $self, $class;
+
+ # Assign function parameters, defaults, and log debug info
+ (
+ my $strOperation,
+ $self->{strId},
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'strId', trace => true},
+ );
+
+ # Initialize the ISA stack
+ $self->{stryIsA} = [COMMON_IO_BASE];
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# error - throw errors
+####################################################################################################################################
+sub error
+{
+ my $self = shift;
+ my $iCode = shift;
+ my $strMessage = shift;
+ my $strDetail = shift;
+
+ logErrorResult($iCode, $strMessage, $strDetail);
+}
+
+####################################################################################################################################
+# isA - get the list of parent classes and add the current class
+####################################################################################################################################
+sub isA
+{
+ my $self = shift;
+
+ unshift(@{$self->{stryIsA}}, $self->className());
+
+ return @{$self->{stryIsA}};
+}
+
+####################################################################################################################################
+# result - retrieve a result from io or a filter
+####################################################################################################################################
+sub result
+{
+ my $self = shift;
+ my $strModule = shift;
+
+ if (!defined($strModule))
+ {
+ return $self->{rhResult};
+ }
+
+ return $self->{rhResult}{$strModule};
+}
+
+####################################################################################################################################
+# resultSet - set a result from io or a filter
+####################################################################################################################################
+sub resultSet
+{
+ my $self = shift;
+ my $strModule = shift;
+ my $xResult = shift;
+
+ $self->{rhResult}{$strModule} = $xResult;
+}
+
+####################################################################################################################################
+# DESTROY - call close()
+####################################################################################################################################
+sub DESTROY {shift->close()}
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub className {blessed(shift)}
+sub id {shift->{strId}}
+
+1;
diff --git a/lib/pgBackRest/Common/Io/Buffered.pm b/lib/pgBackRest/Common/Io/Buffered.pm
new file mode 100644
index 000000000..1825ca3a4
--- /dev/null
+++ b/lib/pgBackRest/Common/Io/Buffered.pm
@@ -0,0 +1,266 @@
+####################################################################################################################################
+# Buffered Handle IO
+####################################################################################################################################
+package pgBackRest::Common::Io::Buffered;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use IO::Select;
+use Time::HiRes qw(gettimeofday);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Base;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::Wait;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant COMMON_IO_BUFFERED => __PACKAGE__;
+ push @EXPORT, qw(COMMON_IO_BUFFERED);
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $self,
+ $iTimeout,
+ $lBufferMax,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'self', trace => true},
+ {name => 'iTimeout', default => 0, trace => true},
+ {name => 'lBufferMax', default => COMMON_IO_BUFFER_MAX, trace => true},
+ );
+
+ # Bless with new class
+ @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+ bless $self, $class;
+
+ # Set write handle so select object is created
+ $self->handleReadSet($self->handleRead());
+
+ # Set variables
+ $self->{iTimeout} = $iTimeout;
+ $self->{lBufferMax} = $lBufferMax;
+
+ # Initialize buffer
+ $self->{tBuffer} = '';
+ $self->{lBufferSize} = 0;
+ $self->{lBufferPos} = 0;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# read - buffered read from a handle with optional blocking
+####################################################################################################################################
+sub read
+{
+ my $self = shift;
+ my $tBufferRef = shift;
+ my $iRequestSize = shift;
+ my $bBlock = shift;
+
+ # Set working variables
+ my $iRemainingSize = $iRequestSize;
+
+ # If there is data left over in the buffer from lineRead then use it
+ my $lBufferRemaining = $self->{lBufferSize} - $self->{lBufferPos};
+
+ if ($lBufferRemaining > 0)
+ {
+ my $iReadSize = $iRequestSize < $lBufferRemaining ? $iRequestSize : $lBufferRemaining;
+
+ $$tBufferRef .= substr($self->{tBuffer}, $self->{lBufferPos}, $iReadSize);
+ $self->{lBufferPos} += $iReadSize;
+
+ $iRemainingSize -= $iReadSize;
+ }
+
+ # If this is a blocking read then loop until all bytes have been read, else error. If not blocking read until the request size
+ # has been met or EOF.
+ my $fTimeStart = gettimeofday();
+ my $fRemaining = $self->timeout();
+
+ while ($iRemainingSize > 0 && $fRemaining > 0)
+ {
+ # Check if the sysread call will block
+ if ($self->{oReadSelect}->can_read($fRemaining))
+ {
+ # Read data into the buffer
+ my $iReadSize = $self->SUPER::read($tBufferRef, $iRemainingSize);
+
+ # Check for EOF
+ if ($iReadSize == 0)
+ {
+ if ($bBlock)
+ {
+ $self->error(ERROR_FILE_READ, "unable to read ${iRequestSize} byte(s) due to EOF from " . $self->id());
+ }
+ else
+ {
+ return $iRequestSize - $iRemainingSize;
+ }
+ }
+
+ # Update remaining size and return when it reaches 0
+ $iRemainingSize -= $iReadSize;
+ }
+
+ # Calculate time remaining before timeout
+ $fRemaining = $self->timeout() - (gettimeofday() - $fTimeStart);
+ };
+
+ # Throw an error if timeout happened before required bytes were read
+ if ($iRemainingSize != 0 && $bBlock)
+ {
+ $self->error(
+ ERROR_FILE_READ, "unable to read ${iRequestSize} byte(s) after " . $self->timeout() . ' second(s) from ' . $self->id());
+ }
+
+ return $iRequestSize - $iRemainingSize;
+}
+
+####################################################################################################################################
+# readLine - read the next lf-terminated line.
+####################################################################################################################################
+sub readLine
+{
+ my $self = shift;
+ my $bIgnoreEOF = shift;
+ my $bError = shift;
+
+ # Try to find the next linefeed
+ my $iLineFeedPos = index($self->{tBuffer}, "\n", $self->{lBufferPos});
+
+ # If no linefeed was found then load more data
+ if ($iLineFeedPos == -1)
+ {
+ my $fRemaining = $self->timeout();
+ my $fTimeStart = gettimeofday();
+
+ # Load data
+ do
+ {
+ # If the buffer already has data and the buffer position is not 0 then trim it so there's room for more data
+ if ($self->{lBufferPos} != 0)
+ {
+ $self->{tBuffer} = substr($self->{tBuffer}, $self->{lBufferPos});
+ $self->{lBufferSize} = $self->{lBufferSize} - $self->{lBufferPos};
+ $self->{lBufferPos} = 0;
+ }
+
+ # Load data into the buffer
+ my $iBufferRead = 0;
+
+ if ($self->{oReadSelect}->can_read($fRemaining))
+ {
+ $iBufferRead = $self->SUPER::read(
+ \$self->{tBuffer},
+ $self->{lBufferSize} >= $self->bufferMax() ? $self->bufferMax() : $self->bufferMax() - $self->{lBufferSize});
+
+ # Check for EOF
+ if ($iBufferRead == 0)
+ {
+ # Return undef if EOF is ignored
+ if (defined($bIgnoreEOF) && $bIgnoreEOF)
+ {
+ return;
+ }
+
+ # Else throw an error
+ $self->error(ERROR_FILE_READ, 'unexpected EOF reading line from ' . $self->id());
+ }
+ }
+
+ # If data was read then check for a linefeed
+ if ($iBufferRead > 0)
+ {
+ $self->{lBufferSize} += $iBufferRead;
+
+ $iLineFeedPos = index($self->{tBuffer}, "\n");
+ }
+
+ # Calculate time remaining before timeout
+ if ($iLineFeedPos == -1)
+ {
+ $fRemaining = $self->timeout() - (gettimeofday() - $fTimeStart);
+ }
+ }
+ while ($iLineFeedPos == -1 && $fRemaining > 0);
+
+ # If not linefeed was found within the timeout throw error
+ if ($iLineFeedPos == -1)
+ {
+ if (!defined($bError) || $bError)
+ {
+ $self->error(
+ ERROR_FILE_READ, 'unable to read line after ' . $self->timeout() . ' second(s) from ' . $self->id());
+ }
+
+ return;
+ }
+ }
+
+ # Return the line that was found and adjust the buffer position
+ my $strLine = substr($self->{tBuffer}, $self->{lBufferPos}, $iLineFeedPos - $self->{lBufferPos});
+ $self->{lBufferPos} = $iLineFeedPos + 1;
+
+ return $strLine;
+}
+
+####################################################################################################################################
+# writeLine - write a string and \n terminate it
+####################################################################################################################################
+sub writeLine
+{
+ my $self = shift;
+ my $strBuffer = shift;
+
+ $strBuffer .= "\n";
+ return $self->SUPER::write(\$strBuffer);
+}
+
+####################################################################################################################################
+# Getters/Setters
+####################################################################################################################################
+sub timeout {shift->{iTimeout}};
+sub bufferMax {shift->{lBufferMax}};
+
+####################################################################################################################################
+# handleReadSet - create a select object when read handle is set
+####################################################################################################################################
+sub handleReadSet
+{
+ my $self = shift;
+ my $fhRead = shift;
+
+ $self->SUPER::handleReadSet($fhRead);
+
+ $self->{oReadSelect} = IO::Select->new();
+ $self->{oReadSelect}->add($self->handleRead());
+}
+
+1;
diff --git a/lib/pgBackRest/Common/Io/Handle.pm b/lib/pgBackRest/Common/Io/Handle.pm
new file mode 100644
index 000000000..d82e67a17
--- /dev/null
+++ b/lib/pgBackRest/Common/Io/Handle.pm
@@ -0,0 +1,157 @@
+####################################################################################################################################
+# Basic Handle IO
+####################################################################################################################################
+package pgBackRest::Common::Io::Handle;
+use parent 'pgBackRest::Common::Io::Base';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant COMMON_IO_HANDLE => __PACKAGE__;
+ push @EXPORT, qw(COMMON_IO_HANDLE);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strId,
+ $fhRead,
+ $fhWrite,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'strId', trace => true},
+ {name => 'fhRead', required => false, trace => true},
+ {name => 'fhWrite', required => false, trace => true},
+ );
+
+ # Create class
+ my $self = $class->SUPER::new($strId);
+ bless $self, $class;
+
+ # Set handles
+ $self->handleReadSet($fhRead) if defined($fhRead);
+ $self->handleWriteSet($fhWrite) if defined($fhWrite);
+
+ # Size tracks number of bytes read and written
+ $self->{lSize} = 0;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# read - read data from handle
+####################################################################################################################################
+sub read
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+ my $iSize = shift;
+
+ # Read the block
+ my $iActualSize;
+
+ eval
+ {
+ $iActualSize = sysread($self->handleRead(), $$rtBuffer, $iSize, defined($$rtBuffer) ? length($$rtBuffer) : 0);
+ return true;
+ }
+ or do
+ {
+ $self->error(ERROR_FILE_READ, 'unable to read from ' . $self->id(), $EVAL_ERROR);
+ };
+
+ # Report any errors
+ # uncoverable branch true - all errors seem to be caught by the handler above but check for error here just in case
+ defined($iActualSize)
+ or $self->error(ERROR_FILE_READ, 'unable to read from ' . $self->id(), $OS_ERROR);
+
+ # Update size
+ $self->{lSize} += $iActualSize;
+
+ return $iActualSize;
+}
+
+####################################################################################################################################
+# write - write data to handle
+####################################################################################################################################
+sub write
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+
+ # Write the block
+ my $iActualSize;
+
+ eval
+ {
+ $iActualSize = syswrite($self->handleWrite(), $$rtBuffer);
+ return true;
+ }
+ or do
+ {
+ $self->error(ERROR_FILE_WRITE, 'unable to write to ' . $self->id(), $EVAL_ERROR);
+ };
+
+ # Report any errors
+ # uncoverable branch true - all errors seem to be caught by the handler above but check for error here just in case
+ defined($iActualSize)
+ or $self->error(ERROR_FILE_WRITE, 'unable to write to ' . $self->id(), $OS_ERROR);
+
+ # Update size
+ $self->{lSize} += $iActualSize;
+
+ return $iActualSize;
+}
+
+####################################################################################################################################
+# close/DESTROY - record read/write size
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ # Set bytes read and written
+ if (defined($self->{lSize}))
+ {
+ $self->resultSet(COMMON_IO_HANDLE, $self->{lSize});
+ undef($self->{lSize});
+ }
+
+ return true;
+}
+
+####################################################################################################################################
+# Getters/Setters
+####################################################################################################################################
+sub handleRead {return shift->{fhHandleRead}}
+sub handleReadSet {my $self = shift; $self->{fhHandleRead} = shift}
+sub handleWrite {return shift->{fhHandleWrite}}
+sub handleWriteSet {my $self = shift; $self->{fhHandleWrite} = shift}
+sub size {shift->{lSize}}
+
+1;
diff --git a/lib/pgBackRest/Common/Io/Process.pm b/lib/pgBackRest/Common/Io/Process.pm
new file mode 100644
index 000000000..854e61738
--- /dev/null
+++ b/lib/pgBackRest/Common/Io/Process.pm
@@ -0,0 +1,188 @@
+####################################################################################################################################
+# Process Excecution, Management, and IO
+####################################################################################################################################
+package pgBackRest::Common::Io::Process;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use IPC::Open3 qw(open3);
+use POSIX qw(:sys_wait_h);
+use Symbol 'gensym';
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Buffered;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::Wait;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant COMMON_IO_PROCESS => __PACKAGE__;
+ push @EXPORT, qw(COMMON_IO_PROCESS);
+
+####################################################################################################################################
+# Amount of time to attempt to retrieve errors when a process terminates unexpectedly
+####################################################################################################################################
+use constant IO_ERROR_TIMEOUT => 5;
+
+####################################################################################################################################
+# new - use open3 to run the command and get the io handles
+####################################################################################################################################
+our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $self,
+ $strCommand,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'self', trace => true},
+ {name => 'strCommand', trace => true},
+ );
+
+ # Bless with new class
+ @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+ bless $self, $class;
+
+ # Use open3 to run the command
+ my ($iProcessId, $fhRead, $fhWrite, $fhReadError);
+ $fhReadError = gensym;
+
+ $iProcessId = IPC::Open3::open3($fhWrite, $fhRead, $fhReadError, $strCommand);
+
+ # Set handles
+ $self->handleReadSet($fhRead);
+ $self->handleWriteSet($fhWrite);
+
+ # Set variables
+ $self->{iProcessId} = $iProcessId;
+ $self->{fhReadError} = $fhReadError;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# error - handle errors
+####################################################################################################################################
+sub error
+{
+ my $self = shift;
+ my $iCode = shift;
+ my $strMessage = shift;
+ my $strDetail = shift;
+ my $bClose = shift;
+
+ if (defined($self->{iProcessId}))
+ {
+ my $oWait = waitInit(defined($iCode) ? IO_ERROR_TIMEOUT : 0);
+
+ do
+ {
+ # Check the result
+ my $iResult = waitpid($self->{iProcessId}, $bClose ? 0 : WNOHANG);
+
+ # Error if the process exited unexpectedly
+ if ($iResult != 0)
+ {
+ # Get the exit status
+ my $iExitStatus = $iResult == -1 ? 255 : ${^CHILD_ERROR_NATIVE} >> 8;
+
+ # Drain the stderr stream
+ my $strError;
+ my $oIoError = new pgBackRest::Common::Io::Buffered(
+ new pgBackRest::Common::Io::Handle($self->id(), $self->{fhReadError}), 5, $self->bufferMax());
+
+ while (defined(my $strLine = $oIoError->readLine(true, false)))
+ {
+ $strError .= "${strLine}\n";
+ }
+
+ delete($self->{iProcessId});
+
+ if (!$bClose || $iExitStatus != 0 || defined($strError))
+ {
+ my $iErrorCode =
+ $iExitStatus >= ERROR_MINIMUM && $iExitStatus <= ERROR_MAXIMUM ? $iExitStatus : ERROR_FILE_READ;
+
+ logErrorResult(
+ $iErrorCode, 'process ' . $self->id() . ' terminated unexpectedly' .
+ ($iExitStatus != 255 ? sprintf(' [%03d]', $iExitStatus) : ''),
+ $strError);
+ }
+ }
+ }
+ while (waitMore($oWait));
+
+ if (defined($iCode))
+ {
+ $self->SUPER::error($iCode, $strMessage, $strDetail);
+ }
+ }
+ else
+ {
+ confess &log(ASSERT, 'cannot call error() after process has been closed');
+ }
+}
+
+####################################################################################################################################
+# Get process id
+####################################################################################################################################
+sub processId
+{
+ my $self = shift;
+
+ return $self->{iProcessId};
+}
+
+####################################################################################################################################
+# writeLine - check for error before writing line
+####################################################################################################################################
+sub writeLine
+{
+ my $self = shift;
+ my $strBuffer = shift;
+
+ # Check if the process has exited abnormally (doesn't seem like we should need this, but the next syswrite does a hard
+ # abort if the remote process has already closed)
+ $self->error();
+
+ return $self->SUPER::writeLine($strBuffer);
+}
+
+####################################################################################################################################
+# close - check if the process terminated on error
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ if (defined($self->{iProcessId}))
+ {
+ $self->error(undef, undef, undef, true);
+
+ # Class parent close
+ $self->SUPER::close();
+ }
+
+ return true;
+}
+
+1;
diff --git a/lib/pgBackRest/Common/Lock.pm b/lib/pgBackRest/Common/Lock.pm
index 9a95ffbb3..040a6ac33 100644
--- a/lib/pgBackRest/Common/Lock.pm
+++ b/lib/pgBackRest/Common/Lock.pm
@@ -16,7 +16,7 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
-use pgBackRest::FileCommon;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# Global lock type and handle
@@ -47,7 +47,7 @@ sub lockFileName
####################################################################################################################################
sub lockPathCreate
{
- filePathCreate(optionGet(OPTION_LOCK_PATH), '770', true, true);
+ storageLocal()->pathCreate(optionGet(OPTION_LOCK_PATH), {strMode => '770', bIgnoreExists => true, bCreateParent => true});
}
####################################################################################################################################
diff --git a/lib/pgBackRest/Common/Log.pm b/lib/pgBackRest/Common/Log.pm
index 654fb9260..e3fd5972d 100644
--- a/lib/pgBackRest/Common/Log.pm
+++ b/lib/pgBackRest/Common/Log.pm
@@ -6,6 +6,7 @@ package pgBackRest::Common::Log;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess longmess);
+use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
@@ -118,13 +119,20 @@ sub logFileSet
# Only open the log file if file logging is enabled
if ($strLogLevelFile ne OFF)
{
- filePathCreate(dirname($strFile), '0770', true, true);
+ # Load Storage::Helper module
+ require pgBackRest::Storage::Helper;
+ pgBackRest::Storage::Helper->import();
+
+ storageLocal()->pathCreate(dirname($strFile), {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
$strFile .= '.log';
$bLogFileExists = -e $strFile ? true : false;
$bLogFileFirst = true;
- $hLogFile = fileOpen($strFile, O_WRONLY | O_CREAT | O_APPEND);
+ if (!sysopen($hLogFile, $strFile, O_WRONLY | O_CREAT | O_APPEND, oct('0660')))
+ {
+ logErrorResult(ERROR_FILE_OPEN, "unable to open log file '${strFile}'", $OS_ERROR);
+ }
# Write out anything that was cached before the file was opened
if (defined($strLogFileCache))
@@ -138,7 +146,6 @@ sub logFileSet
push @EXPORT, qw(logFileSet);
-
####################################################################################################################################
# logBanner
#
@@ -169,10 +176,6 @@ sub logLevelSet
my $strLevelStdErrParam = shift;
my $bLogTimestampParam = shift;
- # Load FileCommon module
- require pgBackRest::FileCommon;
- pgBackRest::FileCommon->import();
-
if (defined($strLevelFileParam))
{
if (!defined($oLogLevelRank{uc($strLevelFileParam)}{rank}))
@@ -690,6 +693,14 @@ sub log
if (!$bSuppressLog)
{
syswrite(*STDOUT, $strMessageFormat);
+
+ # This is here for debugging purposes - it's not clear how best to make it into a switch
+ # if ($strLevel eq ASSERT || $strLevel eq ERROR)
+ # {
+ # my $strStackTrace = longmess() . "\n";
+ # $strStackTrace =~ s/\n/\n /g;
+ # syswrite(*STDOUT, $strStackTrace);
+ # }
}
# If in test mode and this is a test messsage then delay so the calling process has time to read the message
diff --git a/lib/pgBackRest/Common/String.pm b/lib/pgBackRest/Common/String.pm
index aa07087e2..02bba0cca 100644
--- a/lib/pgBackRest/Common/String.pm
+++ b/lib/pgBackRest/Common/String.pm
@@ -33,9 +33,7 @@ sub trim
push @EXPORT, qw(trim);
####################################################################################################################################
-# coalesce
-#
-# Return the first non-null parameter.
+# coalesce - return first defined parameter
####################################################################################################################################
sub coalesce
{
@@ -46,6 +44,8 @@ sub coalesce
return $strParam;
}
}
+
+ return;
}
push @EXPORT, qw(coalesce);
diff --git a/lib/pgBackRest/Config/Config.pm b/lib/pgBackRest/Config/Config.pm
index c1ba26363..76a9f8a57 100644
--- a/lib/pgBackRest/Config/Config.pm
+++ b/lib/pgBackRest/Config/Config.pm
@@ -16,10 +16,9 @@ use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
+use pgBackRest::Common::Io::Base;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
-use pgBackRest::FileCommon;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Version;
####################################################################################################################################
@@ -73,6 +72,14 @@ use constant CMD_VERSION => 'version'
push @EXPORT, qw(CMD_VERSION);
$oCommandHash{&CMD_VERSION} = true;
+####################################################################################################################################
+# DB/BACKUP Constants
+####################################################################################################################################
+use constant DB => 'db';
+ push @EXPORT, qw(DB);
+use constant BACKUP => 'backup';
+ push @EXPORT, qw(BACKUP);
+
####################################################################################################################################
# BACKUP Type Constants
####################################################################################################################################
@@ -83,6 +90,14 @@ use constant BACKUP_TYPE_DIFF => 'diff';
use constant BACKUP_TYPE_INCR => 'incr';
push @EXPORT, qw(BACKUP_TYPE_INCR);
+####################################################################################################################################
+# REPO Type Constants
+####################################################################################################################################
+use constant REPO_TYPE_CIFS => 'cifs';
+ push @EXPORT, qw(REPO_TYPE_CIFS);
+use constant REPO_TYPE_POSIX => 'posix';
+ push @EXPORT, qw(REPO_TYPE_POSIX);
+
####################################################################################################################################
# INFO Output Constants
####################################################################################################################################
@@ -162,6 +177,8 @@ use constant OPTION_RULE_REQUIRED => 'required
push @EXPORT, qw(OPTION_RULE_REQUIRED);
use constant OPTION_RULE_SECTION => 'section';
push @EXPORT, qw(OPTION_RULE_SECTION);
+use constant OPTION_RULE_SECURE => 'secure';
+ push @EXPORT, qw(OPTION_RULE_SECURE);
use constant OPTION_RULE_TYPE => 'type';
push @EXPORT, qw(OPTION_RULE_TYPE);
@@ -258,14 +275,10 @@ use constant OPTION_COMPRESS_LEVEL_NETWORK => 'compress
push @EXPORT, qw(OPTION_COMPRESS_LEVEL_NETWORK);
use constant OPTION_NEUTRAL_UMASK => 'neutral-umask';
push @EXPORT, qw(OPTION_NEUTRAL_UMASK);
-use constant OPTION_REPO_SYNC => 'repo-sync';
- push @EXPORT, qw(OPTION_REPO_SYNC);
use constant OPTION_PROTOCOL_TIMEOUT => 'protocol-timeout';
push @EXPORT, qw(OPTION_PROTOCOL_TIMEOUT);
use constant OPTION_PROCESS_MAX => 'process-max';
push @EXPORT, qw(OPTION_PROCESS_MAX);
-use constant OPTION_REPO_LINK => 'repo-link';
- push @EXPORT, qw(OPTION_REPO_LINK);
# Commands
use constant OPTION_CMD_SSH => 'cmd-ssh';
@@ -276,11 +289,15 @@ use constant OPTION_LOCK_PATH => 'lock-pat
push @EXPORT, qw(OPTION_LOCK_PATH);
use constant OPTION_LOG_PATH => 'log-path';
push @EXPORT, qw(OPTION_LOG_PATH);
-use constant OPTION_REPO_PATH => 'repo-path';
- push @EXPORT, qw(OPTION_REPO_PATH);
use constant OPTION_SPOOL_PATH => 'spool-path';
push @EXPORT, qw(OPTION_SPOOL_PATH);
+# Repository
+use constant OPTION_REPO_PATH => 'repo-path';
+ push @EXPORT, qw(OPTION_REPO_PATH);
+use constant OPTION_REPO_TYPE => 'repo-type';
+ push @EXPORT, qw(OPTION_REPO_TYPE);
+
# Log level
use constant OPTION_LOG_LEVEL_CONSOLE => 'log-level-console';
push @EXPORT, qw(OPTION_LOG_LEVEL_CONSOLE);
@@ -417,12 +434,10 @@ use constant OPTION_DEFAULT_ARCHIVE_TIMEOUT_MIN => WAIT_TIME
use constant OPTION_DEFAULT_ARCHIVE_TIMEOUT_MAX => 86400;
push @EXPORT, qw(OPTION_DEFAULT_ARCHIVE_TIMEOUT_MAX);
-use constant OPTION_DEFAULT_BUFFER_SIZE => 4194304;
+use constant OPTION_DEFAULT_BUFFER_SIZE => COMMON_IO_BUFFER_MAX;
push @EXPORT, qw(OPTION_DEFAULT_BUFFER_SIZE);
use constant OPTION_DEFAULT_BUFFER_SIZE_MIN => 16384;
push @EXPORT, qw(OPTION_DEFAULT_BUFFER_SIZE_MIN);
-use constant OPTION_DEFAULT_BUFFER_SIZE_MAX => 8388608;
- push @EXPORT, qw(OPTION_DEFAULT_BUFFER_SIZE_MAX);
use constant OPTION_DEFAULT_COMPRESS => true;
push @EXPORT, qw(OPTION_DEFAULT_COMPRESS);
@@ -471,6 +486,8 @@ use constant OPTION_DEFAULT_REPO_LINK => true;
push @EXPORT, qw(OPTION_DEFAULT_REPO_LINK);
use constant OPTION_DEFAULT_REPO_PATH => '/var/lib/' . BACKREST_EXE;
push @EXPORT, qw(OPTION_DEFAULT_REPO_PATH);
+use constant OPTION_DEFAULT_REPO_TYPE => REPO_TYPE_POSIX;
+ push @EXPORT, qw(OPTION_DEFAULT_REPO_TYPE);
use constant OPTION_DEFAULT_SPOOL_PATH => '/var/spool/' . BACKREST_EXE;
push @EXPORT, qw(OPTION_DEFAULT_SPOOL_PATH);
use constant OPTION_DEFAULT_PROCESS_MAX => 1;
@@ -1008,7 +1025,20 @@ my %oOptionRule =
&OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
&OPTION_RULE_TYPE => OPTION_TYPE_INTEGER,
&OPTION_RULE_DEFAULT => OPTION_DEFAULT_BUFFER_SIZE,
- &OPTION_RULE_ALLOW_RANGE => [OPTION_DEFAULT_BUFFER_SIZE_MIN, OPTION_DEFAULT_BUFFER_SIZE_MAX],
+ &OPTION_RULE_ALLOW_LIST =>
+ {
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 2 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 4 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 8 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 16 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 32 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 64 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 128 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 256 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 512 => true,
+ &OPTION_DEFAULT_BUFFER_SIZE_MIN * 1024 => true,
+ },
&OPTION_RULE_COMMAND =>
{
&CMD_ARCHIVE_GET => true,
@@ -1196,21 +1226,6 @@ my %oOptionRule =
},
},
- &OPTION_REPO_SYNC =>
- {
- &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
- &OPTION_RULE_TYPE => OPTION_TYPE_BOOLEAN,
- &OPTION_RULE_DEFAULT => OPTION_DEFAULT_REPO_SYNC,
- &OPTION_RULE_NEGATE => true,
- &OPTION_RULE_COMMAND =>
- {
- &CMD_ARCHIVE_PUSH => true,
- &CMD_BACKUP => true,
- &CMD_STANZA_CREATE => true,
- &CMD_STANZA_UPGRADE => true,
- },
- },
-
&OPTION_PROTOCOL_TIMEOUT =>
{
&OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
@@ -1233,17 +1248,6 @@ my %oOptionRule =
}
},
- &OPTION_REPO_LINK =>
- {
- &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
- &OPTION_RULE_TYPE => OPTION_TYPE_BOOLEAN,
- &OPTION_RULE_DEFAULT => OPTION_DEFAULT_REPO_LINK,
- &OPTION_RULE_COMMAND =>
- {
- &CMD_BACKUP => true,
- },
- },
-
&OPTION_REPO_PATH =>
{
&OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
@@ -1267,6 +1271,34 @@ my %oOptionRule =
},
},
+ &OPTION_REPO_TYPE =>
+ {
+ &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
+ &OPTION_RULE_TYPE => OPTION_TYPE_STRING,
+ &OPTION_RULE_DEFAULT => OPTION_DEFAULT_REPO_TYPE,
+ &OPTION_RULE_ALLOW_LIST =>
+ {
+ &REPO_TYPE_CIFS => true,
+ &REPO_TYPE_POSIX => true,
+ },
+ &OPTION_RULE_COMMAND =>
+ {
+ &CMD_ARCHIVE_GET => true,
+ &CMD_ARCHIVE_PUSH => true,
+ &CMD_BACKUP => true,
+ &CMD_CHECK => true,
+ &CMD_EXPIRE => true,
+ &CMD_INFO => true,
+ &CMD_LOCAL => true,
+ &CMD_REMOTE => true,
+ &CMD_RESTORE => true,
+ &CMD_STANZA_CREATE => true,
+ &CMD_STANZA_UPGRADE => true,
+ &CMD_START => true,
+ &CMD_STOP => true,
+ },
+ },
+
&OPTION_SPOOL_PATH =>
{
&OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
@@ -1886,6 +1918,14 @@ my %oOptionRule =
},
&CMD_BACKUP => true,
&CMD_CHECK => true,
+ &CMD_LOCAL =>
+ {
+ &OPTION_RULE_REQUIRED => false
+ },
+ &CMD_REMOTE =>
+ {
+ &OPTION_RULE_REQUIRED => false
+ },
&CMD_RESTORE => true,
&CMD_STANZA_CREATE => true,
&CMD_STANZA_UPGRADE => true,
@@ -1947,6 +1987,38 @@ my %oOptionRule =
},
);
+####################################################################################################################################
+# Process rule defaults
+####################################################################################################################################
+foreach my $strKey (sort(keys(%oOptionRule)))
+{
+ # If the rule is a scalar then copy the entire rule from the referenced option
+ if (!ref($oOptionRule{$strKey}))
+ {
+ $oOptionRule{$strKey} = dclone($oOptionRule{$oOptionRule{$strKey}});
+ }
+
+ # Default type is string
+ if (!defined($oOptionRule{$strKey}{&OPTION_RULE_TYPE}))
+ {
+ $oOptionRule{$strKey}{&OPTION_RULE_TYPE} = OPTION_TYPE_STRING;
+ }
+
+ # If the command section is a scalar then copy the section from the referenced option
+ if (defined($oOptionRule{$strKey}{&OPTION_RULE_COMMAND}) && !ref($oOptionRule{$strKey}{&OPTION_RULE_COMMAND}))
+ {
+ $oOptionRule{$strKey}{&OPTION_RULE_COMMAND} =
+ dclone($oOptionRule{$oOptionRule{$strKey}{&OPTION_RULE_COMMAND}}{&OPTION_RULE_COMMAND});
+ }
+
+ # If the required section is a scalar then copy the section from the referenced option
+ if (defined($oOptionRule{$strKey}{&OPTION_RULE_DEPEND}) && !ref($oOptionRule{$strKey}{&OPTION_RULE_DEPEND}))
+ {
+ $oOptionRule{$strKey}{&OPTION_RULE_DEPEND} =
+ dclone($oOptionRule{$oOptionRule{$strKey}{&OPTION_RULE_DEPEND}}{&OPTION_RULE_DEPEND});
+ }
+}
+
####################################################################################################################################
# Module variables
####################################################################################################################################
@@ -2326,6 +2398,11 @@ sub optionValidate
$oOption{$strOption}{valid} = true;
+ if (defined($oOptionRule{$strOption}{&OPTION_RULE_SECURE}) && $oOptionRule{$strOption}{&OPTION_RULE_SECURE})
+ {
+ $oOption{$strOption}{&OPTION_RULE_SECURE} = true;
+ }
+
# Store the option value
my $strValue = optionValueGet($strOption, $oOptionTest);
@@ -2421,7 +2498,11 @@ sub optionValidate
confess &log(ERROR, "'${strConfigFile}' is not a file", ERROR_FILE_INVALID);
}
- $oConfig = iniParse(fileStringRead($strConfigFile), {bRelaxed => true});
+ # Load Storage::Helper module
+ require pgBackRest::Storage::Helper;
+ pgBackRest::Storage::Helper->import();
+
+ $oConfig = iniParse(${storageLocal->('/')->get($strConfigFile)}, {bRelaxed => true});
}
}
@@ -2742,7 +2823,7 @@ sub configFileValidate
my $bFileValid = true;
- if (!commandTest(CMD_REMOTE))
+ if (!commandTest(CMD_REMOTE) && !commandTest(CMD_LOCAL))
{
foreach my $strSectionKey (keys(%$oConfig))
{
@@ -3208,16 +3289,11 @@ sub commandWrite
$bIncludeConfig = defined($bIncludeConfig) ? $bIncludeConfig : false;
$bIncludeCommand = defined($bIncludeCommand) ? $bIncludeCommand : true;
- # if ($bIncludeConfig && $strExeString ne '')
- # {
- # $strExeString .= ' --no-config';
- # }
-
# Iterate the options to figure out which ones are not default and need to be written out to the new command string
foreach my $strOption (sort(keys(%oOptionRule)))
{
- # Skip the config option if it's already included
- # next if ($bIncludeConfig && $strOption eq OPTION_CONFIG);
+ # Skip option if it is secure and should not be output in logs or the command line
+ next if ($oOption{$strOption}{&OPTION_RULE_SECURE});
# Process any option overrides first
if (defined($$oOptionOverride{$strOption}))
diff --git a/lib/pgBackRest/Config/ConfigHelpData.pm b/lib/pgBackRest/Config/ConfigHelpData.pm
index 7dc4e9230..53595f12b 100644
--- a/lib/pgBackRest/Config/ConfigHelpData.pm
+++ b/lib/pgBackRest/Config/ConfigHelpData.pm
@@ -592,23 +592,6 @@ my $oConfigHelpData =
"edit/check recovery.conf before manually restarting."
},
- # REPO-LINK Option Help
- #---------------------------------------------------------------------------------------------------------------------------
- 'repo-link' =>
- {
- section => 'general',
- summary =>
- "Create convenience symlinks in repository.",
- description =>
- "Creates the convenience link latest in the stanza directory and internal tablespace symlinks in each backup " .
- "directory. The internal tablespace symlinks allow clusters to be brought up manually in-place using " .
- "filesystem snapshots as long as the backup is not compressed.\n" .
- "\n" .
- "This option should be disabled when the repository is located on a filesystem that does not support symlinks. " .
- "No pgBackRest functionality will be affected, but certain manual operations on the repository may be less " .
- "convenient."
- },
-
# REPO-PATH Option Help
#---------------------------------------------------------------------------------------------------------------------------
'repo-path' =>
@@ -625,16 +608,18 @@ my $oConfigHelpData =
"need, though of course requirements will likely change over time as your database evolves."
},
- # REPO-SYNC Option Help
+ # REPO-TYPE Option Help
#---------------------------------------------------------------------------------------------------------------------------
- 'repo-sync' =>
+ 'repo-type' =>
{
section => 'general',
summary =>
- "Sync directories in repository.",
+ "Type of storage used for the repository.",
description =>
- "Syncs directories when writing to the repository. Not all file systems support directory syncs (e.g., NTFS) so " .
- "this option allows them to be disabled."
+ "The following repository types are supported:\n" .
+ "\n" .
+ "* cifs - Like posix, but disables links and directory fsyncs\n" .
+ "* posix - Posix-compliant file systems"
},
# RESUME Option Help
@@ -848,6 +833,7 @@ my $oConfigHelpData =
'neutral-umask' => 'section',
'protocol-timeout' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
@@ -889,7 +875,7 @@ my $oConfigHelpData =
'process-max' => 'section',
'protocol-timeout' => 'section',
'repo-path' => 'section',
- 'repo-sync' => 'section',
+ 'repo-type' => 'section',
'spool-path' => 'section',
'stanza' => 'default'
}
@@ -959,9 +945,8 @@ my $oConfigHelpData =
'online' => 'default',
'process-max' => 'section',
'protocol-timeout' => 'section',
- 'repo-link' => 'section',
'repo-path' => 'section',
- 'repo-sync' => 'section',
+ 'repo-type' => 'section',
'resume' => 'section',
'retention-archive' => 'section',
'retention-archive-type' => 'section',
@@ -1038,6 +1023,7 @@ my $oConfigHelpData =
'online' => 'default',
'protocol-timeout' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
@@ -1068,6 +1054,7 @@ my $oConfigHelpData =
'log-path' => 'section',
'log-timestamp' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
'retention-archive' => 'section',
'retention-archive-type' => 'section',
'retention-diff' => 'section',
@@ -1137,6 +1124,7 @@ my $oConfigHelpData =
'protocol-timeout' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
@@ -1200,6 +1188,7 @@ my $oConfigHelpData =
'protocol-timeout' => 'section',
'recovery-option' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
# SET Option Help
#-------------------------------------------------------------------------------------------------------------------
@@ -1341,7 +1330,7 @@ my $oConfigHelpData =
'online' => 'default',
'protocol-timeout' => 'section',
'repo-path' => 'section',
- 'repo-sync' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
@@ -1384,7 +1373,7 @@ my $oConfigHelpData =
'online' => 'default',
'protocol-timeout' => 'section',
'repo-path' => 'section',
- 'repo-sync' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
@@ -1418,6 +1407,7 @@ my $oConfigHelpData =
'log-path' => 'section',
'log-timestamp' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
@@ -1467,6 +1457,7 @@ my $oConfigHelpData =
'log-path' => 'section',
'log-timestamp' => 'section',
'repo-path' => 'section',
+ 'repo-type' => 'section',
'stanza' => 'default'
}
},
diff --git a/lib/pgBackRest/Db.pm b/lib/pgBackRest/Db.pm
index 6eafc067c..214408d82 100644
--- a/lib/pgBackRest/Db.pm
+++ b/lib/pgBackRest/Db.pm
@@ -20,11 +20,10 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
-use pgBackRest::File;
use pgBackRest::Manifest;
-use pgBackRest::Version;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Version;
####################################################################################################################################
# Backup advisory lock
@@ -87,11 +86,11 @@ sub new
if (defined($self->{iRemoteIdx}))
{
$self->{strDbPath} = optionGet(optionIndex(OPTION_DB_PATH, $self->{iRemoteIdx}));
- $self->{oProtocol} = protocolGet(DB, $self->{iRemoteIdx});
- }
- else
- {
- $self->{oProtocol} = protocolGet(NONE);
+
+ if (!isDbLocal({iRemoteIdx => $self->{iRemoteIdx}}))
+ {
+ $self->{oProtocol} = protocolGet(DB, $self->{iRemoteIdx});
+ }
}
# Return from function and log return values if any
@@ -145,7 +144,7 @@ sub connect
my $bResult = true;
# Run remotely
- if ($self->{oProtocol}->isRemote())
+ if (defined($self->{oProtocol}))
{
# Set bResult to false if undef is returned
$bResult = $self->{oProtocol}->cmdExecute(OP_DB_CONNECT, undef, false, $bWarnOnError) ? true : false;
@@ -246,7 +245,7 @@ sub executeSql
my @stryResult;
# Run remotely
- if ($self->{oProtocol}->isRemote())
+ if (defined($self->{oProtocol}))
{
# Execute the command
@stryResult = @{$self->{oProtocol}->cmdExecute(OP_DB_EXECUTE_SQL, [$strSql, $bIgnoreError, $bResult], $bResult)};
@@ -465,22 +464,14 @@ sub info
#-------------------------------------------------------------------------------------------------------------------------------
if (!defined($self->{info}{$strDbPath}))
{
- # Initialize file object
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- $self->{oProtocol}
- );
-
# Get info from remote
#---------------------------------------------------------------------------------------------------------------------------
- if ($oFile->isRemote(PATH_DB_ABSOLUTE))
+ if (defined($self->{oProtocol}))
{
# Execute the command
($self->{info}{$strDbPath}{strDbVersion}, $self->{info}{$strDbPath}{iDbControlVersion},
$self->{info}{$strDbPath}{iDbCatalogVersion}, $self->{info}{$strDbPath}{ullDbSysId}) =
- $oFile->{oProtocol}->cmdExecute(OP_DB_INFO, [$strDbPath], true);
+ $self->{oProtocol}->cmdExecute(OP_DB_INFO, [$strDbPath], true);
}
# Get info locally
#---------------------------------------------------------------------------------------------------------------------------
diff --git a/lib/pgBackRest/Expire.pm b/lib/pgBackRest/Expire.pm
index e3d2e3ddf..d38fb9edc 100644
--- a/lib/pgBackRest/Expire.pm
+++ b/lib/pgBackRest/Expire.pm
@@ -9,23 +9,21 @@ use Carp qw(confess);
use Exporter qw(import);
use File::Basename qw(dirname);
-use File::Path qw(remove_tree);
use Scalar::Util qw(looks_like_number);
-use pgBackRest::Common::Exception;
-use pgBackRest::Common::Log;
use pgBackRest::Archive::ArchiveCommon;
use pgBackRest::Archive::ArchiveGet;
use pgBackRest::Archive::ArchiveInfo;
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Ini;
+use pgBackRest::Common::Log;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
####################################################################################################################################
# new
@@ -41,14 +39,6 @@ sub new
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
- # Initialize file object
- $self->{oFile} = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(NONE)
- );
-
# Initialize total archive expired
$self->{iArchiveExpireTotal} = 0;
@@ -60,22 +50,6 @@ sub new
);
}
-####################################################################################################################################
-# DESTROY
-####################################################################################################################################
-sub DESTROY
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my ($strOperation) = logDebugParam(__PACKAGE__ . '->DESTROY');
-
- undef($self->{oFile});
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
####################################################################################################################################
# logExpire
#
@@ -128,15 +102,15 @@ sub process
my @stryPath;
- my $oFile = $self->{oFile};
- my $strBackupClusterPath = $oFile->pathGet(PATH_BACKUP_CLUSTER);
+ my $oStorageRepo = storageRepo();
+ my $strBackupClusterPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP);
my $iFullRetention = optionGet(OPTION_RETENTION_FULL, false);
my $iDifferentialRetention = optionGet(OPTION_RETENTION_DIFF, false);
my $strArchiveRetentionType = optionGet(OPTION_RETENTION_ARCHIVE_TYPE, false);
my $iArchiveRetention = optionGet(OPTION_RETENTION_ARCHIVE, false);
# Load the backup.info
- my $oBackupInfo = new pgBackRest::Backup::Info($oFile->pathGet(PATH_BACKUP_CLUSTER));
+ my $oBackupInfo = new pgBackRest::Backup::Info($oStorageRepo->pathGet(STORAGE_REPO_BACKUP));
# Find all the expired full backups
if (defined($iFullRetention))
@@ -158,7 +132,8 @@ sub process
foreach my $strPath ($oBackupInfo->list('^' . $stryPath[$iFullIdx] . '.*'))
{
- $oFile->remove(PATH_BACKUP_CLUSTER, "${strPath}/" . FILE_MANIFEST);
+ $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}/" . FILE_MANIFEST . INI_COPY_EXT);
+ $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}/" . FILE_MANIFEST);
$oBackupInfo->delete($strPath);
if ($strPath ne $stryPath[$iFullIdx])
@@ -204,7 +179,7 @@ sub process
# Remove all differential and incremental backups before the oldest valid differential
if ($strPath lt $stryPath[$iDiffIdx + 1])
{
- $oFile->remove(PATH_BACKUP_CLUSTER, "/${strPath}" . FILE_MANIFEST);
+ $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}" . FILE_MANIFEST);
$oBackupInfo->delete($strPath);
if ($strPath ne $stryPath[$iDiffIdx])
@@ -223,15 +198,14 @@ sub process
$oBackupInfo->save();
# Remove backups from disk
- foreach my $strBackup ($oFile->list(
- PATH_BACKUP_CLUSTER, undef, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))
+ foreach my $strBackup ($oStorageRepo->list(
+ STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))
{
if (!$oBackupInfo->current($strBackup))
{
&log(INFO, "remove expired backup ${strBackup}");
- remove_tree("${strBackupClusterPath}/${strBackup}") > 0
- or confess &log(ERROR, "unable to remove backup ${strBackup}", ERROR_PATH_REMOVE);
+ $oStorageRepo->remove("${strBackupClusterPath}/${strBackup}", {bRecurse => true});
}
}
@@ -264,9 +238,9 @@ sub process
if ($iBackupTotal > 0)
{
- my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE), true);
- my @stryListArchiveDisk = fileList(
- $oFile->pathGet(PATH_BACKUP_ARCHIVE), {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});
+ my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), true);
+ my @stryListArchiveDisk = $oStorageRepo->list(
+ STORAGE_REPO_ARCHIVE, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});
# Make sure the current database versions match between the two files
if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
@@ -288,23 +262,23 @@ sub process
# From the global list of backups to retain, create a list of backups, oldest to newest, associated with this
# archiveId (e.g. 9.4-1)
my @stryLocalBackupRetention = $oBackupInfo->listByArchiveId($strArchiveId,
- $oFile->pathGet(PATH_BACKUP_ARCHIVE), \@stryGlobalBackupRetention, 'reverse');
+ $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \@stryGlobalBackupRetention, 'reverse');
# If no backup to retain was found
if (!@stryLocalBackupRetention)
{
# Get the backup db-id corresponding to this archiveId
- my $iDbHistoryId = $oBackupInfo->backupArchiveDbHistoryId($strArchiveId, $oFile->pathGet(PATH_BACKUP_ARCHIVE));
+ my $iDbHistoryId = $oBackupInfo->backupArchiveDbHistoryId(
+ $strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE));
# If this is not the current database, then delete the archive directory else do nothing since the current
# DB archive directory must not be deleted
if (!defined($iDbHistoryId) || !$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef,
$iDbHistoryId))
{
- my $strFullPath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId);
+ my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}");
- remove_tree($strFullPath) > 0
- or confess &log(ERROR, "unable to remove archive path ${strFullPath}", ERROR_PATH_REMOVE);
+ $oStorageRepo->remove($strFullPath, {bRecurse => true});
&log(INFO, "remove archive path: ${strFullPath}");
}
@@ -370,8 +344,9 @@ sub process
my @stryBackupList = $oBackupInfo->list();
# With the full list of backups, loop through only those associated with this archiveId
- foreach my $strBackup ($oBackupInfo->listByArchiveId($strArchiveId,
- $oFile->pathGet(PATH_BACKUP_ARCHIVE), \@stryBackupList))
+ foreach my $strBackup (
+ $oBackupInfo->listByArchiveId(
+ $strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \@stryBackupList))
{
if ($strBackup le $strArchiveRetentionBackup &&
$oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_ARCHIVE_START))
@@ -400,8 +375,8 @@ sub process
}
# Get all major archive paths (timeline and first 32 bits of LSN)
- foreach my $strPath ($oFile->list(
- PATH_BACKUP_ARCHIVE, $strArchiveId, {strExpression => REGEX_ARCHIVE_DIR_WAL}))
+ foreach my $strPath ($oStorageRepo->list(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveId}", {strExpression => REGEX_ARCHIVE_DIR_WAL}))
{
logDebugMisc($strOperation, "found major WAL path: ${strPath}");
$bRemove = true;
@@ -420,10 +395,9 @@ sub process
# Remove the entire directory if all archive is expired
if ($bRemove)
{
- my $strFullPath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId) . "/${strPath}";
+ my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}") . "/${strPath}";
- remove_tree($strFullPath) > 0
- or confess &log(ERROR, "unable to remove ${strFullPath}", ERROR_PATH_REMOVE);
+ $oStorageRepo->remove($strFullPath, {bRecurse => true});
# Log expire info
logDebugMisc($strOperation, "remove major WAL path: ${strFullPath}");
@@ -435,8 +409,8 @@ sub process
elsif ($strPath le substr($strArchiveExpireMax, 0, 16))
{
# Look for files in the archive directory
- foreach my $strSubPath ($oFile->list(
- PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strPath}", {strExpression => "^[0-F]{24}.*\$"}))
+ foreach my $strSubPath ($oStorageRepo->list(
+ STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strPath}", {strExpression => "^[0-F]{24}.*\$"}))
{
$bRemove = true;
@@ -454,7 +428,7 @@ sub process
# Remove archive log if it is not used in a backup
if ($bRemove)
{
- fileRemove($oFile->pathGet(PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strSubPath}"));
+ $oStorageRepo->remove(STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strSubPath}");
logDebugMisc($strOperation, "remove WAL segment: ${strArchiveId}/${strSubPath}");
diff --git a/lib/pgBackRest/File.pm b/lib/pgBackRest/File.pm
deleted file mode 100644
index 67ee22e44..000000000
--- a/lib/pgBackRest/File.pm
+++ /dev/null
@@ -1,1480 +0,0 @@
-####################################################################################################################################
-# FILE MODULE
-####################################################################################################################################
-package pgBackRest::File;
-
-use strict;
-use warnings FATAL => qw(all);
-use Carp qw(confess);
-use English '-no_match_vars';
-
-use Exporter qw(import);
- our @EXPORT = qw();
-use Fcntl qw(:mode :flock O_RDONLY O_WRONLY O_CREAT);
-use File::Basename qw(dirname basename);
-use File::Copy qw(cp);
-use File::Path qw(make_path remove_tree);
-use File::stat;
-use IO::Handle;
-
-use pgBackRest::Common::Exception;
-use pgBackRest::Common::Log;
-use pgBackRest::Common::String;
-use pgBackRest::Common::Wait;
-use pgBackRest::FileCommon;
-use pgBackRest::Protocol::Common::Common;
-use pgBackRest::Version;
-
-####################################################################################################################################
-# PATH_GET constants
-####################################################################################################################################
-use constant PATH_ABSOLUTE => 'absolute';
- push @EXPORT, qw(PATH_ABSOLUTE);
-use constant PATH_DB => 'db';
- push @EXPORT, qw(PATH_DB);
-use constant PATH_DB_ABSOLUTE => 'db:absolute';
- push @EXPORT, qw(PATH_DB_ABSOLUTE);
-use constant PATH_BACKUP => 'backup';
- push @EXPORT, qw(PATH_BACKUP);
-use constant PATH_BACKUP_ABSOLUTE => 'backup:absolute';
- push @EXPORT, qw(PATH_BACKUP_ABSOLUTE);
-use constant PATH_BACKUP_CLUSTER => 'backup:cluster';
- push @EXPORT, qw(PATH_BACKUP_CLUSTER);
-use constant PATH_BACKUP_TMP => 'backup:tmp';
- push @EXPORT, qw(PATH_BACKUP_TMP);
-use constant PATH_BACKUP_ARCHIVE => 'backup:archive';
- push @EXPORT, qw(PATH_BACKUP_ARCHIVE);
-use constant PATH_BACKUP_ARCHIVE_OUT => 'backup:archive:out';
- push @EXPORT, qw(PATH_BACKUP_ARCHIVE_OUT);
-
-####################################################################################################################################
-# STD pipe constants
-####################################################################################################################################
-use constant PIPE_STDIN => '';
- push @EXPORT, qw(PIPE_STDIN);
-use constant PIPE_STDOUT => '';
- push @EXPORT, qw(PIPE_STDOUT);
-use constant PIPE_STDERR => '';
- push @EXPORT, qw(PIPE_STDERR);
-
-####################################################################################################################################
-# new
-####################################################################################################################################
-sub new
-{
- my $class = shift;
-
- # Create the class hash
- my $self = {};
- bless $self, $class;
-
- # Assign function parameters, defaults, and log debug info
- (
- my $strOperation,
- $self->{strStanza},
- $self->{strRepoPath},
- $self->{oProtocol},
- $self->{strDefaultPathMode},
- $self->{strDefaultFileMode},
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new', \@_,
- {name => 'strStanza', required => false},
- {name => 'strRepoPath'},
- {name => 'oProtocol'},
- {name => 'strDefaultPathMode', default => '0750'},
- {name => 'strDefaultFileMode', default => '0640'},
- );
-
- # Default compression extension to gz
- $self->{strCompressExtension} = COMPRESS_EXT;
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'self', value => $self}
- );
-}
-
-####################################################################################################################################
-# pathTypeGet
-####################################################################################################################################
-sub pathTypeGet
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strType
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->pathTypeGet', \@_,
- {name => 'strType', trace => true}
- );
-
- my $strPath;
-
- # If absolute type
- if ($strType eq PATH_ABSOLUTE)
- {
- $strPath = PATH_ABSOLUTE;
- }
- # If db type
- elsif ($strType =~ /^db(\:.*){0,1}/)
- {
- $strPath = PATH_DB;
- }
- # Else if backup type
- elsif ($strType =~ /^backup(\:.*){0,1}/)
- {
- $strPath = PATH_BACKUP;
- }
- # else error when path type not recognized
- else
- {
- confess &log(ASSERT, "no known path types in '${strType}'");
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strPath', value => $strPath, trace => true}
- );
-}
-
-####################################################################################################################################
-# pathGet
-####################################################################################################################################
-sub pathGet
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strType, # Base type of the path to get (PATH_DB_ABSOLUTE, PATH_BACKUP_TMP, etc)
- $strFile, # File to append to the base path (can include a path as well)
- $bTemp # Return the temp file for this path type - only some types have temp files
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->pathGet', \@_,
- {name => 'strType', trace => true},
- {name => 'strFile', required => false, trace => true},
- {name => 'bTemp', default => false, trace => true}
- );
-
- # Path to be returned
- my $strPath;
-
- # Is this an absolute path type?
- my $bAbsolute = $strType =~ /.*absolute.*/;
-
- # Make sure a temp file is valid for this type and file
- if ($bTemp)
- {
- # Only allow temp files for PATH_BACKUP_ARCHIVE, PATH_BACKUP_ARCHIVE_OUT, PATH_BACKUP_TMP and any absolute path
- if (!($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_ARCHIVE_OUT || $strType eq PATH_BACKUP_TMP || $bAbsolute))
- {
- confess &log(ASSERT, "temp file not supported for path type '${strType}'");
- }
-
- # The file must be defined
- if (!defined($strFile))
- {
- confess &log(ASSERT, 'strFile must be defined when temp file specified');
- }
- }
-
- # Get absolute path
- if ($bAbsolute)
- {
- # File must defined when the path is absolute since in effect there is no path
- if (!defined($strFile))
- {
- confess &log(ASSERT, 'strFile must be defined for absolute path');
- }
-
- # Make sure that the file starts with /, otherwise it will actually be relative
- if ($strFile !~ /^\/.*/)
- {
- confess &log(ASSERT, "absolute path ${strType}:${strFile} must start with /");
- }
- }
- # Else get backup path
- elsif ($strType eq PATH_BACKUP)
- {
- $strPath = $self->{strRepoPath};
- }
- # Else process path types that require a stanza
- else
- {
- # All paths in this section will in the repo path
- $strPath = $self->{strRepoPath};
-
- # Make sure the stanza is defined since remaining path types require it
- if (!defined($self->{strStanza}))
- {
- confess &log(ASSERT, 'strStanza not defined');
- }
-
- # Get the backup tmp path
- if ($strType eq PATH_BACKUP_TMP)
- {
- $strPath .= "/temp/$self->{strStanza}.tmp";
- }
- # Else get archive paths
- elsif ($strType eq PATH_BACKUP_ARCHIVE_OUT || $strType eq PATH_BACKUP_ARCHIVE)
- {
- $strPath .= "/archive/$self->{strStanza}";
-
- # Get archive path
- if ($strType eq PATH_BACKUP_ARCHIVE)
- {
- # If file is not defined nothing further to do
- if (defined($strFile))
- {
- my $strArchiveId = (split('/', $strFile))[0];
-
- # If file is defined after archive id path is split out
- if (defined((split('/', $strFile))[1]))
- {
- $strPath .= "/${strArchiveId}";
- $strFile = (split('/', $strFile))[1];
-
- # If this is a WAL segment then put it into a subdirectory
- if (substr(basename($strFile), 0, 24) =~ /^([0-F]){24}$/)
- {
- $strPath .= '/' . substr($strFile, 0, 16);
- }
- }
- }
- }
- # Else get archive out path
- else
- {
- $strPath .= '/out';
- }
- }
- # Else get backup cluster
- elsif ($strType eq PATH_BACKUP_CLUSTER)
- {
- $strPath .= "/backup/$self->{strStanza}";
- }
- # Else error when path type not recognized
- else
- {
- confess &log(ASSERT, "no known path types in '${strType}'");
- }
- }
-
- # Combine path and file
- $strPath .= (defined($strFile) ? (defined($strPath) ? '/' : '') . $strFile : '');
-
- # Add temp extension
- $strPath .= $bTemp ? '.' . BACKREST_EXE . '.tmp' : '';
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strPath', value => $strPath, trace => true}
- );
-}
-
-####################################################################################################################################
-# isRemote
-#
-# Determine whether the path type is remote
-####################################################################################################################################
-sub isRemote
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->isRemote', \@_,
- {name => 'strPathType', trace => true}
- );
-
- my $bRemote = $self->{oProtocol}->isRemote() && $self->{oProtocol}->remoteTypeTest($self->pathTypeGet($strPathType));
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bRemote', value => $bRemote, trace => true}
- );
-}
-
-####################################################################################################################################
-# linkCreate
-####################################################################################################################################
-sub linkCreate
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strSourcePathType,
- $strSourceFile,
- $strDestinationPathType,
- $strDestinationFile,
- $bHard,
- $bRelative,
- $bPathCreate
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->linkCreate', \@_,
- {name => 'strSourcePathType'},
- {name => 'strSourceFile'},
- {name => 'strDestinationPathType'},
- {name => 'strDestinationFile'},
- {name => 'bHard', default => false},
- {name => 'bRelative', default => false},
- {name => 'bPathCreate', default => true}
- );
-
- # Source and destination path types must be the same (e.g. both PATH_DB or both PATH_BACKUP, etc.)
- if ($self->pathTypeGet($strSourcePathType) ne $self->pathTypeGet($strDestinationPathType))
- {
- confess &log(ASSERT, 'path types must be equal in link create');
- }
-
- # Generate source and destination files
- my $strSource = $self->pathGet($strSourcePathType, $strSourceFile);
- my $strDestination = $self->pathGet($strDestinationPathType, $strDestinationFile);
-
- # Run remotely
- if ($self->isRemote($strSourcePathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
- # Run locally
- else
- {
- # If the destination path is backup and does not exist, create it
- # ??? This should only happen when the link create errors
- if ($bPathCreate && $self->pathTypeGet($strDestinationPathType) eq PATH_BACKUP)
- {
- filePathCreate(dirname($strDestination), undef, true);
- }
-
- unless (-e $strSource)
- {
- if (-e $strSource . ".$self->{strCompressExtension}")
- {
- $strSource .= ".$self->{strCompressExtension}";
- $strDestination .= ".$self->{strCompressExtension}";
- }
- else
- {
- # Error when a hardlink will be created on a missing file
- if ($bHard)
- {
- confess &log(ASSERT, "unable to find ${strSource}(.$self->{strCompressExtension}) for link");
- }
- }
- }
-
- # Generate relative path if requested
- if ($bRelative)
- {
- # Determine how much of the paths are common
- my @strySource = split('/', $strSource);
- my @stryDestination = split('/', $strDestination);
-
- while (defined($strySource[0]) && defined($stryDestination[0]) && $strySource[0] eq $stryDestination[0])
- {
- shift(@strySource);
- shift(@stryDestination);
- }
-
- # Add relative path sections
- $strSource = '';
-
- for (my $iIndex = 0; $iIndex < @stryDestination - 1; $iIndex++)
- {
- $strSource .= '../';
- }
-
- # Add path to source
- $strSource .= join('/', @strySource);
-
- logDebugMisc
- (
- $strOperation, 'apply relative path',
- {name => 'strSource', value => $strSource, trace => true}
- );
- }
-
- if ($bHard)
- {
- link($strSource, $strDestination)
- or confess &log(ERROR, "unable to create hardlink from ${strSource} to ${strDestination}");
- }
- else
- {
- symlink($strSource, $strDestination)
- or confess &log(ERROR, "unable to create symlink from ${strSource} to ${strDestination}");
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-####################################################################################################################################
-# move
-#
-# Moves a file locally or remotely.
-####################################################################################################################################
-sub move
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strSourcePathType,
- $strSourceFile,
- $strDestinationPathType,
- $strDestinationFile,
- $bDestinationPathCreate,
- $bPathSync,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->move', \@_,
- {name => 'strSourcePathType'},
- {name => 'strSourceFile', required => false},
- {name => 'strDestinationPathType'},
- {name => 'strDestinationFile'},
- {name => 'bDestinationPathCreate', default => false},
- {name => 'bPathSync', default => false},
- );
-
- # Source and destination path types must be the same
- if ($self->pathTypeGet($strSourcePathType) ne $self->pathTypeGet($strSourcePathType))
- {
- confess &log(ASSERT, 'source and destination path types must be equal');
- }
-
- # Set operation variables
- my $strPathOpSource = $self->pathGet($strSourcePathType, $strSourceFile);
- my $strPathOpDestination = $self->pathGet($strDestinationPathType, $strDestinationFile);
-
- # Run remotely
- if ($self->isRemote($strSourcePathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
- # Run locally
- else
- {
- fileMove($strPathOpSource, $strPathOpDestination, $bDestinationPathCreate, $bPathSync);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation
- );
-}
-
-####################################################################################################################################
-# compress
-####################################################################################################################################
-sub compress
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strFile,
- $bRemoveSource
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->compress', \@_,
- {name => 'strPathType'},
- {name => 'strFile'},
- {name => 'bRemoveSource', default => true}
- );
-
- # Set operation variables
- my $strPathOp = $self->pathGet($strPathType, $strFile);
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
- # Run locally
- else
- {
- # Use copy to compress the file
- $self->copy($strPathType, $strFile, $strPathType, "${strFile}.gz", false, true);
-
- # Remove the old file
- if ($bRemoveSource)
- {
- fileRemove($strPathOp);
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation
- );
-}
-
-####################################################################################################################################
-# pathCreate
-#
-# Creates a path locally or remotely.
-####################################################################################################################################
-sub pathCreate
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strPath,
- $strMode,
- $bIgnoreExists,
- $bCreateParents
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->pathCreate', \@_,
- {name => 'strPathType'},
- {name => 'strPath', required => false},
- {name => 'strMode', default => '0750'},
- {name => 'bIgnoreExists', default => false},
- {name => 'bCreateParents', default => false}
- );
-
- # Set operation variables
- my $strPathOp = $self->pathGet($strPathType, $strPath);
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- $self->{oProtocol}->cmdExecute(OP_FILE_PATH_CREATE, [$strPathOp, $strMode, $bIgnoreExists, $bCreateParents]);
- }
- # Run locally
- else
- {
- filePathCreate($strPathOp, $strMode, $bIgnoreExists, $bCreateParents);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation
- );
-}
-
-####################################################################################################################################
-# pathSync
-####################################################################################################################################
-sub pathSync
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strPath,
- $bRecursive,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->pathSync', \@_,
- {name => 'strPathType'},
- {name => 'strPath', required => false},
- {name => 'bRecursive', default => false},
- );
-
- # Remote not implemented
- if ($self->isRemote($strPathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
-
- # Sync all paths in the tree
- if ($bRecursive)
- {
- my $oManifest = $self->manifest($strPathType, $strPath);
-
- # Iterate all files in the manifest
- foreach my $strFile (sort(keys(%{$oManifest})))
- {
- # Only sync if this is a directory
- if ($oManifest->{$strFile}{type} eq 'd')
- {
- # If current directory
- if ($strFile eq '.')
- {
- $self->pathSync($strPathType, $strPath);
- }
- # Else a subdirectory
- else
- {
- $self->pathSync($strPathType, (defined($strPath) ? "${strPath}/" : '') . $strFile);
- }
- }
- }
- }
- # Only sync the specified path
- else
- {
- filePathSync($self->pathGet($strPathType, $strPath));
- }
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-####################################################################################################################################
-# exists
-#
-# Checks for the existence of a file, but does not imply that the file is readable/writeable.
-#
-# Return: true if file exists, false otherwise
-####################################################################################################################################
-sub exists
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strPath
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->exists', \@_,
- {name => 'strPathType'},
- {name => 'strPath', required => false}
- );
-
- # Set operation variables
- my $strPathOp = $self->pathGet($strPathType, $strPath);
- my $bExists;
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- $bExists = $self->{oProtocol}->cmdExecute(OP_FILE_EXISTS, [$strPathOp], true);
- }
- # Run locally
- else
- {
- $bExists = fileExists($strPathOp);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bExists', value => $bExists}
- );
-}
-
-####################################################################################################################################
-# remove
-####################################################################################################################################
-sub remove
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strPath,
- $bTemp,
- $bIgnoreMissing,
- $bPathSync,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->remove', \@_,
- {name => 'strPathType'},
- {name => 'strPath'},
- {name => 'bTemp', required => false},
- {name => 'bIgnoreMissing', default => true},
- {name => 'bPathSync', default => false},
- );
-
- # Set operation variables
- my $strPathOp = $self->pathGet($strPathType, $strPath, $bTemp);
- my $bRemoved = true;
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
- # Run locally
- else
- {
- $bRemoved = fileRemove($strPathOp, $bIgnoreMissing, $bPathSync);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bRemoved', value => $bRemoved}
- );
-}
-
-####################################################################################################################################
-# hash
-####################################################################################################################################
-sub hash
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strFile,
- $bCompressed,
- $strHashType
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->hash', \@_,
- {name => 'strPathType'},
- {name => 'strFile'},
- {name => 'bCompressed', required => false},
- {name => 'strHashType', required => false}
- );
-
- my ($strHash) = $self->hashSize($strPathType, $strFile, $bCompressed, $strHashType);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strHash', value => $strHash, trace => true}
- );
-}
-
-####################################################################################################################################
-# hashSize
-####################################################################################################################################
-sub hashSize
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strFile,
- $bCompressed,
- $strHashType
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->hashSize', \@_,
- {name => 'strPathType'},
- {name => 'strFile'},
- {name => 'bCompressed', default => false},
- {name => 'strHashType', default => 'sha1'}
- );
-
- # Set operation variables
- my $strFileOp = $self->pathGet($strPathType, $strFile);
- my $strHash;
- my $iSize = 0;
-
- if ($self->isRemote($strPathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
- else
- {
- ($strHash, $iSize) = fileHashSize($strFileOp, $bCompressed, $strHashType, $self->{oProtocol});
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strHash', value => $strHash},
- {name => 'iSize', value => $iSize}
- );
-}
-
-####################################################################################################################################
-# owner
-####################################################################################################################################
-sub owner
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strFile,
- $strUser,
- $strGroup
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->owner', \@_,
- {name => 'strPathType'},
- {name => 'strFile'},
- {name => 'strUser', required => false},
- {name => 'strGroup', required => false}
- );
-
- # Set operation variables
- my $strFileOp = $self->pathGet($strPathType, $strFile);
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- confess &log(ASSERT, "${strOperation}: remote operation not supported");
- }
- # Run locally
- else
- {
- my $iUserId;
- my $iGroupId;
-
- # If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that
- # both user and group be set.
- if (!(defined($strUser) && defined($strGroup)))
- {
- my $oStat = fileStat($strFileOp);
-
- if (!defined($strUser))
- {
- $iUserId = $oStat->uid;
- }
-
- if (!defined($strGroup))
- {
- $iGroupId = $oStat->gid;
- }
- }
-
- # Lookup user if specified
- if (defined($strUser))
- {
- $iUserId = getpwnam($strUser);
-
- if (!defined($iUserId))
- {
- confess &log(ERROR, "user '${strUser}' does not exist", ERROR_USER_MISSING);
- }
- }
-
- # Lookup group if specified
- if (defined($strGroup))
- {
- $iGroupId = getgrnam($strGroup);
-
- if (!defined($iGroupId))
- {
- confess &log(ERROR, "group '${strGroup}' does not exist", ERROR_GROUP_MISSING);
- }
- }
-
- # Set ownership on the file
- if (!chown($iUserId, $iGroupId, $strFileOp))
- {
- my $strError = $!;
-
- if (fileExists($strFileOp))
- {
- confess &log(ERROR,
- "unable to set ownership for '${strFileOp}'" . (defined($strError) ? ": $strError" : ''), ERROR_FILE_OWNER);
- }
-
- confess &log(ERROR, "${strFile} does not exist", ERROR_FILE_MISSING);
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation
- );
-}
-
-####################################################################################################################################
-# list
-####################################################################################################################################
-sub list
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strPath,
- $strExpression,
- $strSortOrder,
- $bIgnoreMissing
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->list', \@_,
- {name => 'strPathType'},
- {name => 'strPath', required => false},
- {name => 'strExpression', optional => true},
- {name => 'strSortOrder', optional => true, default => 'forward'},
- {name => 'bIgnoreMissing', optional => true, default => false}
- );
-
- # Set operation variables
- my $strPathOp = $self->pathGet($strPathType, $strPath);
- my @stryFileList;
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- @stryFileList = $self->{oProtocol}->cmdExecute(
- OP_FILE_LIST,
- [$strPathOp, {strExpression => $strExpression, strSortOrder => $strSortOrder, bIgnoreMissing => $bIgnoreMissing}]);
- }
- # Run locally
- else
- {
- @stryFileList = fileList(
- $strPathOp, {strExpression => $strExpression, strSortOrder => $strSortOrder, bIgnoreMissing => $bIgnoreMissing});
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'stryFileList', value => \@stryFileList}
- );
-}
-
-####################################################################################################################################
-# wait
-#
-# Wait until the next second. This is done in the file object because it must be performed on whichever side the db is on, local or
-# remote. This function is used to make sure that no files are copied in the same second as the manifest is created. The reason is
-# that the db might modify they file again in the same second as the copy and that change will not be visible to a subsequent
-# incremental backup using timestamp/size to determine deltas.
-####################################################################################################################################
-sub wait
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $bWait
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->wait', \@_,
- {name => 'strPathType'},
- {name => 'bWait', default => true}
- );
-
- # Second when the function was called
- my $lTimeBegin;
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- $lTimeBegin = $self->{oProtocol}->cmdExecute(OP_FILE_WAIT, [$bWait], true);
- }
- # Run locally
- else
- {
- # Wait the remainder of the current second
- $lTimeBegin = waitRemainder($bWait);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'lTimeBegin', value => $lTimeBegin, trace => true}
- );
-}
-
-####################################################################################################################################
-# manifest
-#
-# Builds a path/file manifest starting with the base path and including all subpaths. The manifest contains all the information
-# needed to perform a backup or a delta with a previous backup.
-####################################################################################################################################
-sub manifest
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPathType,
- $strPath,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->manifest', \@_,
- {name => 'strPathType'},
- {name => 'strPath', required => false},
- );
-
- # Set operation variables
- my $strPathOp = $self->pathGet($strPathType, $strPath);
- my $hManifest;
-
- # Run remotely
- if ($self->isRemote($strPathType))
- {
- $hManifest = $self->{oProtocol}->cmdExecute(OP_FILE_MANIFEST, [$strPathOp], true);
- }
- # Run locally
- else
- {
- $hManifest = fileManifest($strPathOp);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'hManifest', value => $hManifest, trace => true}
- );
-}
-
-####################################################################################################################################
-# copy
-#
-# Copies a file from one location to another:
-#
-# * source and destination can be local or remote
-# * wire and output compression/decompression are supported
-# * intermediate temp files are used to prevent partial copies
-# * modification time, mode, and ownership can be set on destination file
-# * destination path can optionally be created
-####################################################################################################################################
-sub copy
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strSourcePathType,
- $strSourceFile,
- $strDestinationPathType,
- $strDestinationFile,
- $bSourceCompressed,
- $bDestinationCompress,
- $bIgnoreMissingSource,
- $lModificationTime,
- $strMode,
- $bDestinationPathCreate,
- $strUser,
- $strGroup,
- $bAppendChecksum,
- $bPathSync,
- $strExtraFunction,
- $rExtraParam,
- $bTempFile,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->copy', \@_,
- {name => 'strSourcePathType'},
- {name => 'strSourceFile', required => false},
- {name => 'strDestinationPathType'},
- {name => 'strDestinationFile', required => false},
- {name => 'bSourceCompressed', default => false},
- {name => 'bDestinationCompress', default => false},
- {name => 'bIgnoreMissingSource', default => false},
- {name => 'lModificationTime', required => false},
- {name => 'strMode', default => fileModeDefaultGet()},
- {name => 'bDestinationPathCreate', default => false},
- {name => 'strUser', required => false},
- {name => 'strGroup', required => false},
- {name => 'bAppendChecksum', default => false},
- {name => 'bPathSync', default => false},
- {name => 'strExtraFunction', required => false},
- {name => 'rExtraParam', required => false},
- {name => 'bTempFile', default => true},
- );
-
- # Set working variables
- my $bSourceRemote = $self->isRemote($strSourcePathType) || $strSourcePathType eq PIPE_STDIN;
- my $bDestinationRemote = $self->isRemote($strDestinationPathType) || $strDestinationPathType eq PIPE_STDOUT;
- my $strSourceOp = $strSourcePathType eq PIPE_STDIN ?
- $strSourcePathType : $self->pathGet($strSourcePathType, $strSourceFile);
- my $strDestinationOp = $strDestinationPathType eq PIPE_STDOUT ?
- $strDestinationPathType : $self->pathGet($strDestinationPathType, $strDestinationFile);
- my $strDestinationTmpOp =
- $strDestinationPathType eq PIPE_STDOUT ? undef :
- ($bTempFile ? $self->pathGet($strDestinationPathType, $strDestinationFile, true) :
- $self->pathGet($strDestinationPathType, $strDestinationFile));
- my $fnExtra = defined($strExtraFunction) ?
- eval("\\&${strExtraFunction}") : undef; ## no critic (BuiltinFunctions::ProhibitStringyEval)
-
- # Checksum and size variables
- my $strChecksum = undef;
- my $iFileSize = undef;
- my $rExtra = undef;
- my $bResult = true;
-
- # Temp file is required if checksum will be appended
- if ($bAppendChecksum && !$bTempFile)
- {
- confess &log(ASSERT, 'bTempFile must be true when bAppendChecksum is true');
- }
-
- # Open the source and destination files (if needed)
- my $hSourceFile;
- my $hDestinationFile;
-
- if (!$bSourceRemote)
- {
- if (!sysopen($hSourceFile, $strSourceOp, O_RDONLY))
- {
- my $strError = $!;
- my $iErrorCode = ERROR_FILE_READ;
-
- if ($!{ENOENT})
- {
- # $strError = 'file is missing';
- $iErrorCode = ERROR_FILE_MISSING;
-
- if ($bIgnoreMissingSource && $strDestinationPathType ne PIPE_STDOUT)
- {
- return false, undef, undef;
- }
- }
-
- $strError = "cannot open source file ${strSourceOp}: " . $strError;
-
- if ($strSourcePathType eq PATH_ABSOLUTE)
- {
- if ($strDestinationPathType eq PIPE_STDOUT)
- {
- $self->{oProtocol}->binaryXferAbort();
- }
- }
-
- confess &log(ERROR, $strError, $iErrorCode);
- }
- }
-
- if (!$bDestinationRemote)
- {
- my $iCreateFlag = O_WRONLY | O_CREAT;
-
- # Open the destination temp file
- if (!sysopen($hDestinationFile, $strDestinationTmpOp, $iCreateFlag, oct($strMode)))
- {
- if ($bDestinationPathCreate)
- {
- filePathCreate(dirname($strDestinationTmpOp), undef, true, true);
- }
-
- if (!$bDestinationPathCreate || !sysopen($hDestinationFile, $strDestinationTmpOp, $iCreateFlag, oct($strMode)))
- {
- my $strError = "unable to open ${strDestinationTmpOp}: " . $!;
- my $iErrorCode = ERROR_FILE_READ;
-
- if (!fileExists(dirname($strDestinationTmpOp)))
- {
- $strError = dirname($strDestinationTmpOp) . ' destination path does not exist';
- $iErrorCode = ERROR_FILE_MISSING;
- }
-
- if (!($bDestinationPathCreate && $iErrorCode == ERROR_FILE_MISSING))
- {
- confess &log(ERROR, $strError, $iErrorCode);
- }
- }
- }
-
- # Now lock the file to be sure nobody else is operating on it
- if (!flock($hDestinationFile, LOCK_EX | LOCK_NB))
- {
- confess &log(ERROR, "unable to acquire exclusive lock on ${strDestinationTmpOp}", ERROR_LOCK_ACQUIRE);
- }
-
- # Set user and/or group if required
- if (defined($strUser) || defined($strGroup))
- {
- $self->owner(PATH_ABSOLUTE, $strDestinationTmpOp, $strUser, $strGroup);
- }
- }
-
- # If source or destination are remote
- if ($bSourceRemote || $bDestinationRemote)
- {
- # Build the command and open the local file
- my $hIn,
- my $hOut;
- my $strRemote;
- my $strRemoteOp;
- my $bController = false;
-
- # If source is remote and destination is local
- if ($bSourceRemote && !$bDestinationRemote)
- {
- $hOut = $hDestinationFile;
- $strRemoteOp = OP_FILE_COPY_OUT;
- $strRemote = 'in';
-
- if ($strSourcePathType ne PIPE_STDIN)
- {
- $self->{oProtocol}->cmdWrite($strRemoteOp,
- [$strSourceOp, undef, $bSourceCompressed, $bDestinationCompress, undef, undef, undef, undef, undef, undef,
- undef, undef, $strExtraFunction, $rExtraParam, $bTempFile]);
-
- $bController = true;
- }
- }
- # Else if source is local and destination is remote
- elsif (!$bSourceRemote && $bDestinationRemote)
- {
- $hIn = $hSourceFile;
- $strRemoteOp = OP_FILE_COPY_IN;
- $strRemote = 'out';
-
- if ($strDestinationPathType ne PIPE_STDOUT)
- {
- $self->{oProtocol}->cmdWrite(
- $strRemoteOp,
- [undef, $strDestinationOp, $bSourceCompressed, $bDestinationCompress, undef, undef, $strMode,
- $bDestinationPathCreate, $strUser, $strGroup, $bAppendChecksum, $bPathSync, $strExtraFunction,
- $rExtraParam, $bTempFile]);
-
- $bController = true;
- }
- }
- # Else source and destination are remote
- else
- {
- $strRemoteOp = OP_FILE_COPY;
-
- $self->{oProtocol}->cmdWrite(
- $strRemoteOp,
- [$strSourceOp, $strDestinationOp, $bSourceCompressed, $bDestinationCompress, $bIgnoreMissingSource, undef,
- $strMode, $bDestinationPathCreate, $strUser, $strGroup, $bAppendChecksum, $bPathSync, $strExtraFunction,
- $rExtraParam, $bTempFile]);
-
- $bController = true;
- }
-
- # Transfer the file (skip this for copies where both sides are remote)
- if ($strRemoteOp ne OP_FILE_COPY)
- {
- ($strChecksum, $iFileSize, $rExtra) =
- $self->{oProtocol}->binaryXfer($hIn, $hOut, $strRemote, $bSourceCompressed, $bDestinationCompress, undef, $fnExtra,
- $rExtraParam);
- }
-
- # If this is the controlling process then wait for OK from remote
- if ($bController)
- {
- # Test for an error when reading output
- eval
- {
- ($bResult, my $strResultChecksum, my $iResultFileSize, my $rResultExtra) =
- $self->{oProtocol}->outputRead(true, $bIgnoreMissingSource);
-
- # Check the result of the remote call
- if ($bResult)
- {
- # If the operation was purely remote, get checksum/size
- if ($strRemoteOp eq OP_FILE_COPY ||
- $strRemoteOp eq OP_FILE_COPY_IN && $bSourceCompressed && !$bDestinationCompress)
- {
- # Checksum shouldn't already be set
- if (defined($strChecksum) || defined($iFileSize))
- {
- confess &log(ASSERT, "checksum and size are already defined, but shouldn't be");
- }
-
- $strChecksum = $strResultChecksum;
- $iFileSize = $iResultFileSize;
- $rExtra = $rResultExtra;
- }
- }
-
- return true;
- }
- # If there is an error then evaluate
- or do
- {
- my $oException = $EVAL_ERROR;
-
- # Ignore error if source file was missing and missing file exception was returned and bIgnoreMissingSource is set
- if ($bIgnoreMissingSource && $strRemote eq 'in' && exceptionCode($oException) == ERROR_FILE_MISSING)
- {
- close($hDestinationFile)
- or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
- fileRemove($strDestinationTmpOp);
-
- $bResult = false;
- }
- else
- {
- confess $oException;
- }
- };
- }
- }
- # Else this is a local operation
- else
- {
- # If the source is not compressed and the destination is then compress
- if (!$bSourceCompressed && $bDestinationCompress)
- {
- ($strChecksum, $iFileSize, $rExtra) =
- $self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'out', false, true, false, $fnExtra, $rExtraParam);
- }
- # If the source is compressed and the destination is not then decompress
- elsif ($bSourceCompressed && !$bDestinationCompress)
- {
- ($strChecksum, $iFileSize, $rExtra) =
- $self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'in', true, false, false, $fnExtra, $rExtraParam);
- }
- # Else both sides are compressed, so copy capturing checksum
- elsif ($bSourceCompressed)
- {
- ($strChecksum, $iFileSize, $rExtra) =
- $self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'out', true, true, false, $fnExtra, $rExtraParam);
- }
- else
- {
- ($strChecksum, $iFileSize, $rExtra) =
- $self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'in', false, true, false, $fnExtra, $rExtraParam);
- }
- }
-
- if ($bResult)
- {
- # Close the source file (if local)
- if (defined($hSourceFile))
- {
- close($hSourceFile) or confess &log(ERROR, "cannot close file ${strSourceOp}");
- }
-
- # Sync and close the destination file (if local)
- if (defined($hDestinationFile))
- {
- $hDestinationFile->sync()
- or confess &log(ERROR, "unable to sync ${strDestinationTmpOp}", ERROR_FILE_SYNC);
-
- close($hDestinationFile)
- or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
- }
-
- # Checksum and file size should be set if the destination is not remote
- if (!(!$bSourceRemote && $bDestinationRemote && $bSourceCompressed) &&
- (!defined($strChecksum) || !defined($iFileSize)))
- {
- confess &log(ASSERT, 'checksum or file size not set');
- }
-
- # Where the destination is local, set mode, modification time, and perform move to final location
- if (!$bDestinationRemote)
- {
- # Set the file modification time if required
- if (defined($lModificationTime))
- {
- utime($lModificationTime, $lModificationTime, $strDestinationTmpOp)
- or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
- }
-
- # Replace checksum in destination filename (if exists)
- if ($bAppendChecksum)
- {
- # Replace destination filename
- if ($bDestinationCompress)
- {
- $strDestinationOp =
- substr($strDestinationOp, 0, length($strDestinationOp) - length($self->{strCompressExtension}) - 1) .
- '-' . $strChecksum . '.' . $self->{strCompressExtension};
- }
- else
- {
- $strDestinationOp .= '-' . $strChecksum;
- }
- }
-
- # Move the file from tmp to final destination
- if ($bTempFile)
- {
- fileMove($strDestinationTmpOp, $strDestinationOp, $bDestinationPathCreate, $bPathSync);
- }
- # Else sync path if requested
- elsif ($bPathSync)
- {
- filePathSync(dirname($strDestinationTmpOp));
- }
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bResult', value => $bResult, trace => true},
- {name => 'strChecksum', value => $strChecksum, trace => true},
- {name => 'iFileSize', value => $iFileSize, trace => true},
- {name => '$rExtra', value => $rExtra, trace => true},
- );
-}
-
-1;
diff --git a/lib/pgBackRest/FileCommon.pm b/lib/pgBackRest/FileCommon.pm
deleted file mode 100644
index b038a05b6..000000000
--- a/lib/pgBackRest/FileCommon.pm
+++ /dev/null
@@ -1,1143 +0,0 @@
-####################################################################################################################################
-# FILE COMMON MODULE
-####################################################################################################################################
-package pgBackRest::FileCommon;
-
-use strict;
-use warnings FATAL => qw(all);
-use Carp qw(confess);
-use English '-no_match_vars';
-
-use Digest::SHA;
-use Exporter qw(import);
- our @EXPORT = qw();
-use Fcntl qw(:mode :flock O_RDONLY O_WRONLY O_CREAT O_TRUNC);
-use File::Basename qw(dirname basename);
-use File::Path qw(make_path);
-use File::stat;
-use IO::Handle;
-
-use pgBackRest::Common::Exception;
-use pgBackRest::Common::Log;
-use pgBackRest::Version;
-
-####################################################################################################################################
-# Default modes
-####################################################################################################################################
-my $strPathModeDefault = '0750';
-my $strFileModeDefault = '0640';
-
-####################################################################################################################################
-# Compression extension
-####################################################################################################################################
-use constant COMPRESS_EXT => 'gz';
- push @EXPORT, qw(COMPRESS_EXT);
-
-####################################################################################################################################
-# fileExists
-#
-# Check if a path or file exists.
-####################################################################################################################################
-sub fileExists
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileExists', \@_,
- {name => 'strFile', required => true, trace => true}
- );
-
- # Working variables
- my $bExists = true;
-
- # Stat the file/path to determine if it exists
- my $oStat = lstat($strFile);
-
- # Evaluate error
- if (!defined($oStat))
- {
- my $strError = $!;
-
- # If the error is not entry missing, then throw error
- if (!$!{ENOENT})
- {
- confess &log(ERROR, "unable to read ${strFile}" . (defined($strError) ? ": $strError" : ''), ERROR_FILE_OPEN);
- }
-
- $bExists = false;
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bExists', value => $bExists, trace => true}
- );
-}
-
-push @EXPORT, qw(fileExists);
-
-####################################################################################################################################
-# fileHash
-#
-# Get the file hash and size.
-####################################################################################################################################
-sub fileHash
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile,
- $bCompressed,
- $strHashType
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileHash', \@_,
- {name => 'strFile', trace => true},
- {name => 'bCompressed', default => false, trace => true},
- {name => 'strHashType', default => 'sha1', trace => true}
- );
-
- # Working variables
- my ($strHash) = fileHashSize($strFile, $bCompressed, $strHashType);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strHash', value => $strHash, trace => true}
- );
-}
-
-push @EXPORT, qw(fileHash);
-
-####################################################################################################################################
-# fileHashSize
-#
-# Get the file hash and size.
-####################################################################################################################################
-sub fileHashSize
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile,
- $bCompressed,
- $strHashType,
- $oProtocol
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileHashSize', \@_,
- {name => 'strFile', trace => true},
- {name => 'bCompressed', default => false, trace => true},
- {name => 'strHashType', default => 'sha1', trace => true},
- {name => 'oProtocol', required => false, trace => true}
- );
-
- # Working variables
- my $strHash;
- my $iSize = 0;
- my $hFile;
-
- if (!sysopen($hFile, $strFile, O_RDONLY))
- {
- my $strError = $!;
-
- # If file exists then throw the error
- if (fileExists($strFile))
- {
- confess &log(ERROR, "unable to open ${strFile}" . (defined($strError) ? ": $strError" : ''), ERROR_FILE_OPEN);
- }
-
- confess &log(ERROR, "${strFile} does not exist", ERROR_FILE_MISSING);
- }
-
- my $oSHA = Digest::SHA->new($strHashType);
-
- if ($bCompressed)
- {
- # ??? Not crazy about pushing the protocol object in here. Probably binaryXfer() should be refactored into a standalone
- # function in this file.
- if (!defined($oProtocol))
- {
- confess &log(ASSERT, "oProtocol must be provided to hash compressed file");
- }
-
- ($strHash, $iSize) =
- $oProtocol->binaryXfer($hFile, 'none', 'in', true, false, false);
- }
- else
- {
- my $iBlockSize;
- my $tBuffer;
-
- do
- {
- # Read a block from the file
- $iBlockSize = sysread($hFile, $tBuffer, 4194304);
-
- if (!defined($iBlockSize))
- {
- confess &log(ERROR, "${strFile} could not be read: " . $!, ERROR_FILE_READ);
- }
-
- $iSize += $iBlockSize;
- $oSHA->add($tBuffer);
- }
- while ($iBlockSize > 0);
-
- $strHash = $oSHA->hexdigest();
- }
-
- close($hFile);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strHash', value => $strHash, trace => true},
- {name => 'iSize', value => $iSize, trace => true}
- );
-}
-
-push @EXPORT, qw(fileHashSize);
-
-####################################################################################################################################
-# fileLinkDestination
-#
-# Return the destination of a symlink.
-####################################################################################################################################
-sub fileLinkDestination
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strLink,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileLinkDestination', \@_,
- {name => 'strFile', trace => true},
- );
-
- # Get link destination
- my $strLinkDestination = readlink($strLink);
-
- # Check for errors
- if (!defined($strLinkDestination))
- {
- logErrorResult(
- $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to get destination for link ${strLink}", $OS_ERROR);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strLinkDestination', value => $strLinkDestination, trace => true}
- );
-}
-
-push @EXPORT, qw(fileLinkDestination);
-
-####################################################################################################################################
-# fileList
-#
-# List a directory with filters and ordering.
-####################################################################################################################################
-sub fileList
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath,
- $strExpression,
- $strSortOrder,
- $bIgnoreMissing,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileList', \@_,
- {name => 'strPath', trace => true},
- {name => 'strExpression', optional => true, trace => true},
- {name => 'strSortOrder', optional => true, default => 'forward', trace => true},
- {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
- );
-
- # Working variables
- my @stryFileList;
- my $hPath;
-
- # Attempt to open the path
- if (opendir($hPath, $strPath))
- {
- @stryFileList = grep(!/^(\.)|(\.\.)$/i, readdir($hPath));
- close($hPath);
-
- # Apply expression if defined
- if (defined($strExpression))
- {
- @stryFileList = grep(/$strExpression/i, @stryFileList);
- }
-
- # Reverse sort
- if ($strSortOrder eq 'reverse')
- {
- @stryFileList = sort {$b cmp $a} @stryFileList;
- }
- # Normal sort
- else
- {
- @stryFileList = sort @stryFileList;
- }
- }
- # Else process errors
- else
- {
- my $strError = $!;
-
- # Ignore the error is the file is missing and missing files should be ignored
- if (!($!{ENOENT} && $bIgnoreMissing))
- {
- confess &log(ERROR, "unable to read ${strPath}" . (defined($strError) ? ": $strError" : ''), ERROR_PATH_OPEN);
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'stryFileList', value => \@stryFileList, trace => true}
- );
-}
-
-push @EXPORT, qw(fileList);
-
-####################################################################################################################################
-# fileManifest
-#
-# Generate a complete list of all directories/links/files in a directory/subdirectories.
-####################################################################################################################################
-sub fileManifest
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileManifest', \@_,
- {name => 'strPath', trace => true},
- );
-
- # Generate the manifest
- my $hManifest = {};
- fileManifestRecurse($strPath, undef, 0, $hManifest);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'hManifest', value => $hManifest, trace => true}
- );
-}
-
-push @EXPORT, qw(fileManifest);
-
-sub fileManifestRecurse
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath,
- $strSubPath,
- $iDepth,
- $hManifest,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileManifestRecurse', \@_,
- {name => 'strPath', trace => true},
- {name => 'strSubPath', required => false, trace => true},
- {name => 'iDepth', default => 0, trace => true},
- {name => 'hManifest', required => false, trace => true},
- );
-
- # Set operation and debug strings
- my $strPathRead = $strPath . (defined($strSubPath) ? "/${strSubPath}" : '');
- my $hPath;
- my $strFilter;
-
- # If this is the top level stat the path to discover if it is actually a file
- if ($iDepth == 0 && !S_ISDIR((fileStat($strPathRead))->mode))
- {
- $strFilter = basename($strPathRead);
- $strPathRead = dirname($strPathRead);
- }
-
- # Get a list of all files in the path (including .)
- my @stryFileList = fileList($strPathRead, {bIgnoreMissing => $iDepth != 0});
- unshift(@stryFileList, '.');
- my $hFileStat = fileManifestList($strPathRead, \@stryFileList);
-
- # Loop through all subpaths/files in the path
- foreach my $strFile (keys(%{$hFileStat}))
- {
- # Skip this file if it does not match the filter
- if (defined($strFilter) && $strFile ne $strFilter)
- {
- next;
- }
-
- my $strManifestFile = $iDepth == 0 ? $strFile : ($strSubPath . ($strFile eq qw(.) ? '' : "/${strFile}"));
- $hManifest->{$strManifestFile} = $hFileStat->{$strFile};
-
- # Recurse into directories
- if ($hManifest->{$strManifestFile}{type} eq 'd' && $strFile ne qw(.))
- {
- fileManifestRecurse($strPath, $strManifestFile, $iDepth + 1, $hManifest);
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-sub fileManifestList
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath,
- $stryFile,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileManifestList', \@_,
- {name => 'strPath', trace => true},
- {name => 'stryFile', trace => true},
- );
-
- my $hFileStat = {};
-
- foreach my $strFile (@{$stryFile})
- {
- $hFileStat->{$strFile} = fileManifestStat("${strPath}" . ($strFile eq qw(.) ? '' : "/${strFile}"));
-
- if (!defined($hFileStat->{$strFile}))
- {
- delete($hFileStat->{$strFile});
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'hFileStat', value => $hFileStat, trace => true}
- );
-}
-
-sub fileManifestStat
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileManifestRecurse', \@_,
- {name => 'strFile', trace => true},
- );
-
- # Stat the path/file, ignoring any that are missing
- my $oStat = fileStat($strFile, true);
-
- # Generate file data if stat succeeded (i.e. file exists)
- my $hFile;
-
- if (defined($oStat))
- {
- # Check for regular file
- if (S_ISREG($oStat->mode))
- {
- $hFile->{type} = 'f';
-
- # Get size
- $hFile->{size} = $oStat->size;
-
- # Get modification time
- $hFile->{modification_time} = $oStat->mtime;
- }
- # Check for directory
- elsif (S_ISDIR($oStat->mode))
- {
- $hFile->{type} = 'd';
- }
- # Check for link
- elsif (S_ISLNK($oStat->mode))
- {
- $hFile->{type} = 'l';
- $hFile->{link_destination} = fileLinkDestination($strFile);
- }
- # Not a recognized type
- else
- {
- confess &log(ERROR, "${strFile} is not of type directory, file, or link", ERROR_FILE_INVALID);
- }
-
- # Get user name
- $hFile->{user} = getpwuid($oStat->uid);
-
- # Get group name
- $hFile->{group} = getgrgid($oStat->gid);
-
- # Get mode
- if ($hFile->{type} ne 'l')
- {
- $hFile->{mode} = sprintf('%04o', S_IMODE($oStat->mode));
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'hFile', value => $hFile, trace => true}
- );
-}
-
-####################################################################################################################################
-# fileMode
-#
-# Set the file mode.
-####################################################################################################################################
-sub fileMode
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile,
- $strMode,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileMode', \@_,
- {name => 'strFile', trace => true},
- {name => 'strMode', default => $strFileModeDefault, trace => true},
- );
-
- # Change mode
- if(!chmod(oct($strMode), $strFile))
- {
- my $strError = $!;
-
- # If file exists then throw the error
- if (fileExists($strFile))
- {
- confess &log(ERROR, "unable to chmod ${strFile}" . (defined($strError) ? ": $strError" : ''), ERROR_FILE_MODE);
- }
-
- confess &log(ERROR, "${strFile} does not exist", ERROR_FILE_MISSING);
- }
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(fileMode);
-
-####################################################################################################################################
-# fileModeDefaultGet
-#
-# Get the default mode to be used when creating files.
-####################################################################################################################################
-sub fileModeDefaultGet
-{
- # Assign function parameters, defaults, and log debug info
- my ($strOperation) = logDebugParam(__PACKAGE__ . '::fileModeDefaultGet');
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strFileModeDefault', value => $strFileModeDefault, trace => true}
- );
-}
-
-push @EXPORT, qw(fileModeDefaultGet);
-
-####################################################################################################################################
-# fileModeDefaultSet
-#
-# Set the default mode to be used when creating files.
-####################################################################################################################################
-sub fileModeDefaultSet
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strMode,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileModeDefaultSet', \@_,
- {name => 'strMode', trace => true},
- );
-
- $strFileModeDefault = $strMode;
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(fileModeDefaultSet);
-
-####################################################################################################################################
-# fileMove
-#
-# Move a file.
-####################################################################################################################################
-sub fileMove
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strSourceFile,
- $strDestinationFile,
- $bDestinationPathCreate,
- $bPathSync,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileMove', \@_,
- {name => 'strSourceFile', trace => true},
- {name => 'strDestinationFile', trace => true},
- {name => 'bDestinationPathCreate', default => false, trace => true},
- {name => 'bPathSync', default => false, trace => true},
- );
-
- # Get source and destination paths
- my $strSourcePath = dirname($strSourceFile);
- my $strDestinationPath = dirname($strDestinationFile);
-
- # Move the file
- if (!rename($strSourceFile, $strDestinationFile))
- {
- my $strError = $!;
- my $bError = true;
-
- # If the destination path does not exist and can be created then create it
- if ($bDestinationPathCreate && !fileExists($strDestinationPath))
- {
- $bError = false;
-
- filePathCreate(dirname($strDestinationFile), undef, true, true, $bPathSync);
-
- # Try the rename again and store the error if it fails
- if (!rename($strSourceFile, $strDestinationFile))
- {
- $strError = $!;
- $bError = true;
- }
- }
-
- # If there was an error then raise it
- if ($bError)
- {
- confess &log(ERROR, "unable to move file ${strSourceFile} to ${strDestinationFile}" .
- (defined($strError) ? ": $strError" : ''), ERROR_FILE_MOVE);
- }
- }
-
- # Sync path(s) if requested
- if ($bPathSync)
- {
- # Always sync the destination directory
- filePathSync(dirname($strDestinationFile));
-
- # If the source and destination directories are not the same then sync the source directory
- if (dirname($strSourceFile) ne dirname($strDestinationFile))
- {
- filePathSync(dirname($strSourceFile));
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(fileMove);
-
-####################################################################################################################################
-# fileOpen
-#
-# Open a file.
-####################################################################################################################################
-sub fileOpen
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile,
- $lFlags,
- $strMode,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileOpen', \@_,
- {name => 'strFile', trace => true},
- {name => 'lFlags', trace => true},
- {name => 'strMode', default => $strFileModeDefault, trace => true},
- );
-
- my $hFile;
-
- if (!sysopen($hFile, $strFile, $lFlags, oct($strMode)))
- {
- my $strError = $!;
-
- # If file exists then throw the error
- if (fileExists($strFile))
- {
- confess &log(ERROR, "unable to open ${strFile}" . (defined($strError) ? ": $strError" : ''), ERROR_FILE_OPEN);
- }
-
- confess &log(ERROR, "${strFile} does not exist", ERROR_FILE_MISSING);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'hFile', value => $hFile, trace => true}
- );
-}
-
-push @EXPORT, qw(fileOpen);
-
-####################################################################################################################################
-# filePathSync
-#
-# Sync a directory.
-####################################################################################################################################
-sub filePathSync
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::filePathSync', \@_,
- {name => 'strPath', trace => true}
- );
-
- open(my $hPath, "<", $strPath)
- or confess &log(ERROR, "unable to open '${strPath}' for sync", ERROR_PATH_OPEN);
- open(my $hPathDup, ">&", $hPath)
- or confess &log(ERROR, "unable to duplicate '${strPath}' handle for sync", ERROR_PATH_OPEN);
-
- $hPathDup->sync
- or confess &log(ERROR, "unable to sync '${strPath}'", ERROR_PATH_SYNC);
-
- close($hPathDup);
- close($hPath);
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(filePathSync);
-
-####################################################################################################################################
-# fileRemove
-#
-# Remove a file from the file system.
-####################################################################################################################################
-sub fileRemove
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath,
- $bIgnoreMissing,
- $bPathSync,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileRemove', \@_,
- {name => 'strPath', trace => true},
- {name => 'bIgnoreMissing', default => false, trace => true},
- {name => 'bPathSync', default => false, trace => true},
- );
-
- # Working variables
- my $bRemoved = true;
-
- # Remove the file
- if (unlink($strPath) != 1)
- {
- $bRemoved = false;
- my $strError = $!;
-
- # If path exists then throw the error
- if (fileExists($strPath))
- {
- confess &log(ERROR, "unable to remove ${strPath}" . (defined($strError) ? ": $strError" : ''), ERROR_FILE_OPEN);
- }
- # Else throw an error unless missing paths are ignored
- elsif (!$bIgnoreMissing)
- {
- confess &log(ERROR, "${strPath} does not exist", ERROR_FILE_MISSING);
- }
- }
-
- # Sync parent directory if requested
- if ($bRemoved && $bPathSync)
- {
- filePathSync(dirname($strPath));
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bRemoved', value => $bRemoved, trace => true}
- );
-}
-
-push @EXPORT, qw(fileRemove);
-
-####################################################################################################################################
-# fileStat
-#
-# Stat a file.
-####################################################################################################################################
-sub fileStat
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFile,
- $bIgnoreMissing,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileStat', \@_,
- {name => 'strFile', trace => true},
- {name => 'bIgnoreMissing', default => false, trace => true},
- );
-
- # Stat the file/path
- my $oStat = lstat($strFile);
-
- # Check for errors
- if (!defined($oStat))
- {
- if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
- {
- logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to stat ${strFile}", $OS_ERROR);
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'oStat', value => $oStat, trace => true}
- );
-}
-
-push @EXPORT, qw(fileStat);
-
-####################################################################################################################################
-# fileStringRead
-#
-# Read the specified file as a string.
-####################################################################################################################################
-sub fileStringRead
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFileName
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileStringRead', \@_,
- {name => 'strFileName', trace => true}
- );
-
- # Open the file for writing
- sysopen(my $hFile, $strFileName, O_RDONLY)
- or confess &log(ERROR, "unable to open ${strFileName}");
-
- # Read the string
- my $iBytesRead;
- my $iBytesTotal = 0;
- my $strContent;
-
- do
- {
- $iBytesRead = sysread($hFile, $strContent, 65536, $iBytesTotal);
-
- if (!defined($iBytesRead))
- {
- confess &log(ERROR, "unable to read string from ${strFileName}: $!", ERROR_FILE_READ);
- }
-
- $iBytesTotal += $iBytesRead;
- }
- while ($iBytesRead != 0);
-
- close($hFile);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strContent', value => $strContent, trace => true}
- );
-}
-
-push @EXPORT, qw(fileStringRead);
-
-####################################################################################################################################
-# fileStringWrite
-#
-# Create/overwrite a file with a string.
-####################################################################################################################################
-sub fileStringWrite
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strFileName,
- $strContent,
- $bSync,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::fileStringWrite', \@_,
- {name => 'strFileName', trace => true},
- {name => 'strContent', trace => true, required => false},
- {name => 'bSync', default => true, trace => true},
- );
-
- # Generate temp filename
- my $strFileNameTemp = $strFileName . '.' . BACKREST_EXE . '.tmp';
-
- # Open file for writing
- my $hFile = fileOpen($strFileNameTemp, O_WRONLY | O_CREAT | O_TRUNC, $strFileModeDefault);
-
- # Write content
- if (defined($strContent) && length($strContent) > 0)
- {
- syswrite($hFile, $strContent)
- or confess &log(ERROR, "unable to write string to ${strFileName}: $!", ERROR_FILE_WRITE);
- }
-
- # Sync file
- $hFile->sync() if $bSync;
-
- # Close file
- close($hFile);
-
- # Move file to final location
- fileMove($strFileNameTemp, $strFileName, undef, $bSync);
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(fileStringWrite);
-
-####################################################################################################################################
-# pathAbsolute
-#
-# Generate an absolute path from an absolute base path and a relative path.
-####################################################################################################################################
-sub pathAbsolute
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strBasePath,
- $strPath
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::pathAbsolute', \@_,
- {name => 'strBasePath', trace => true},
- {name => 'strPath', trace => true}
- );
-
- # Working variables
- my $strAbsolutePath;
-
- # If the path is already absolute
- if (index($strPath, '/') == 0)
- {
- $strAbsolutePath = $strPath;
- }
- # Else make it absolute using the base path
- else
- {
- # Make sure the absolute path is really absolute
- if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1)
- {
- confess &log(ERROR, "${strBasePath} is not an absolute path", ERROR_PATH_TYPE);
- }
-
- while (index($strPath, '..') == 0)
- {
- $strBasePath = dirname($strBasePath);
- $strPath = substr($strPath, 2);
-
- if (index($strPath, '/') == 0)
- {
- $strPath = substr($strPath, 1);
- }
- }
-
- $strAbsolutePath = "${strBasePath}/${strPath}";
- }
-
- # Make sure the result is really an absolute path
- if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1)
- {
- confess &log(ERROR, "result ${strAbsolutePath} was not an absolute path", ERROR_PATH_TYPE);
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'strAbsolutePath', value => $strAbsolutePath, trace => true}
- );
-}
-
-push @EXPORT, qw(pathAbsolute);
-
-####################################################################################################################################
-# pathModeDefaultSet
-#
-# Set the default mode to be used when creating paths.
-####################################################################################################################################
-sub pathModeDefaultSet
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strMode
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::pathModeDefaultSet', \@_,
- {name => 'strMode', trace => true},
- );
-
- $strPathModeDefault = $strMode;
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(pathModeDefaultSet);
-
-####################################################################################################################################
-# filePathCreate
-#
-# Create a path.
-####################################################################################################################################
-sub filePathCreate
-{
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strPath,
- $strMode,
- $bIgnoreExists,
- $bCreateParents,
- $bPathSync,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '::filePathCreate', \@_,
- {name => 'strPath', trace => true},
- {name => 'strMode', default => $strPathModeDefault, trace => true},
- {name => 'bIgnoreExists', default => false, trace => true},
- {name => 'bCreateParents', default => false, trace => true},
- {name => 'bPathSync', default => false, trace => true},
- );
-
- # Determine if parent directory exists
- my $strParentPath = dirname($strPath);
-
- if (!fileExists($strParentPath))
- {
- if (!$bCreateParents)
- {
- confess &log(ERROR, "unable to create ${strPath} because parent path does not exist", ERROR_PATH_CREATE);
- }
-
- filePathCreate($strParentPath, $strMode, $bIgnoreExists, $bCreateParents, $bPathSync);
- }
-
- # Create the path
- if (!mkdir($strPath, oct($strMode)))
- {
- # Get the error as a string
- my $strError = $OS_ERROR . '';
-
- # Error if not ignoring existence of the path
- if (!($bIgnoreExists && fileExists($strPath)))
- {
- confess &log(ERROR, "unable to create ${strPath}: " . $OS_ERROR, ERROR_PATH_CREATE);
- }
- }
-
- # Sync path if requested
- if ($bPathSync)
- {
- filePathSync($strParentPath);
- }
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
-push @EXPORT, qw(filePathCreate);
-
-1;
diff --git a/lib/pgBackRest/Info.pm b/lib/pgBackRest/Info.pm
index 658c38ca4..aa0892dbe 100644
--- a/lib/pgBackRest/Info.pm
+++ b/lib/pgBackRest/Info.pm
@@ -11,18 +11,19 @@ use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
+use pgBackRest::Backup::Common;
+use pgBackRest::Backup::Info;
use pgBackRest::Common::Log;
use pgBackRest::Common::Ini;
use pgBackRest::Common::String;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# Info constants
@@ -99,16 +100,8 @@ sub process
# Get stanza if specified
my $strStanza = optionTest(OPTION_STANZA) ? optionGet(OPTION_STANZA) : undef;
- # Create the file object
- my $oFile = new pgBackRest::File
- (
- $strStanza,
- optionGet(OPTION_REPO_PATH),
- protocolGet(BACKUP)
- );
-
# Get the stanza list with all info
- my $oyStanzaList = $self->stanzaList($oFile, $strStanza);
+ my $oyStanzaList = $self->stanzaList($strStanza);
if (optionTest(OPTION_OUTPUT, INFO_OUTPUT_TEXT))
{
@@ -120,7 +113,7 @@ sub process
}
else
{
- syswrite(*STDOUT, 'No stanzas exist in ' . $oFile->pathGet(PATH_BACKUP) . ".\n");
+ syswrite(*STDOUT, 'No stanzas exist in ' . storageRepo()->pathGet() . ".\n");
}
}
elsif (optionTest(OPTION_OUTPUT, INFO_OUTPUT_JSON))
@@ -325,27 +318,25 @@ sub stanzaList
my
(
$strOperation,
- $oFile,
$strStanza
) =
logDebugParam
(
__PACKAGE__ . '->stanzaList', \@_,
- {name => 'oFile'},
{name => 'strStanza', required => false}
);
my @oyStanzaList;
# Run remotely
- if ($oFile->isRemote(PATH_BACKUP))
+ if (!isRepoLocal())
{
- @oyStanzaList = @{$oFile->{oProtocol}->cmdExecute(OP_INFO_STANZA_LIST, [$strStanza], true)};
+ @oyStanzaList = @{protocolGet(BACKUP)->cmdExecute(OP_INFO_STANZA_LIST, [$strStanza], true)};
}
# Run locally
else
{
- my @stryStanza = $oFile->list(PATH_BACKUP, CMD_BACKUP, {bIgnoreMissing => true});
+ my @stryStanza = storageRepo()->list(CMD_BACKUP, {bIgnoreMissing => true});
foreach my $strStanzaFound (@stryStanza)
{
@@ -357,7 +348,7 @@ sub stanzaList
my $oStanzaInfo = {};
$$oStanzaInfo{&INFO_STANZA_NAME} = $strStanzaFound;
($$oStanzaInfo{&INFO_BACKUP_SECTION_BACKUP}, $$oStanzaInfo{&INFO_BACKUP_SECTION_DB}) =
- $self->backupList($oFile, $strStanzaFound);
+ $self->backupList($strStanzaFound);
# If there are no backups then set status to no backup
if (@{$$oStanzaInfo{&INFO_BACKUP_SECTION_BACKUP}} == 0)
@@ -394,20 +385,20 @@ sub stanzaList
# With multiple DB versions, the backup.info history-id may not be the same as archive.info history-id, so the
# archive path must be built by retrieving the archive id given the db version and system id of the backup
- my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo(optionGet(OPTION_REPO_PATH) . "/" . $strArchiveStanzaPath);
+ my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo(storageRepo()->pathGet($strArchiveStanzaPath));
my $strArchiveId = $oArchiveInfo->archiveId({strDbVersion => $hDbInfo->{&INFO_KEY_VERSION},
ullDbSysId => $hDbInfo->{&INFO_KEY_SYSTEM_ID}});
my $strArchivePath = "archive/${strStanzaFound}/${strArchiveId}";
- if ($oFile->exists(PATH_BACKUP, $strArchivePath))
+ if (storageRepo()->pathExists($strArchivePath))
{
- my @stryWalMajor = $oFile->list(PATH_BACKUP, $strArchivePath, {strExpression => '^[0-F]{16}$'});
+ my @stryWalMajor = storageRepo()->list($strArchivePath, {strExpression => '^[0-F]{16}$'});
# Get first WAL segment
foreach my $strWalMajor (@stryWalMajor)
{
- my @stryWalFile = $oFile->list(
- PATH_BACKUP, "${strArchivePath}/${strWalMajor}",
+ my @stryWalFile = storageRepo()->list(
+ "${strArchivePath}/${strWalMajor}",
{strExpression => "^[0-F]{24}-[0-f]{40}(\\." . COMPRESS_EXT . "){0,1}\$"});
if (@stryWalFile > 0)
@@ -420,8 +411,8 @@ sub stanzaList
# Get last WAL segment
foreach my $strWalMajor (sort({$b cmp $a} @stryWalMajor))
{
- my @stryWalFile = $oFile->list(
- PATH_BACKUP, "${strArchivePath}/${strWalMajor}",
+ my @stryWalFile = storageRepo()->list(
+ "${strArchivePath}/${strWalMajor}",
{strExpression => "^[0-F]{24}-[0-f]{40}(\\." . COMPRESS_EXT . "){0,1}\$", strSortOrder => 'reverse'});
if (@stryWalFile > 0)
@@ -481,18 +472,16 @@ sub backupList
my
(
$strOperation,
- $oFile,
$strStanza
) =
logDebugParam
(
__PACKAGE__ . '->backupList', \@_,
- {name => 'oFile'},
{name => 'strStanza'}
);
# Load the backup.info but do not attempt to validate it or confirm it's existence
- my $oBackupInfo = new pgBackRest::Backup::Info($oFile->pathGet(PATH_BACKUP, CMD_BACKUP . "/${strStanza}"), false, false);
+ my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet(CMD_BACKUP . "/${strStanza}"), false, false, {bIgnoreMissing => true});
# Build the db list
my @oyDbList;
diff --git a/lib/pgBackRest/Manifest.pm b/lib/pgBackRest/Manifest.pm
index 190197608..2342fa4d3 100644
--- a/lib/pgBackRest/Manifest.pm
+++ b/lib/pgBackRest/Manifest.pm
@@ -18,9 +18,11 @@ use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
+use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
+use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# File/path constants
@@ -29,6 +31,8 @@ use constant PATH_BACKUP_HISTORY => 'backup.h
push @EXPORT, qw(PATH_BACKUP_HISTORY);
use constant FILE_MANIFEST => 'backup.manifest';
push @EXPORT, qw(FILE_MANIFEST);
+use constant FILE_MANIFEST_COPY => FILE_MANIFEST . INI_COPY_EXT;
+ push @EXPORT, qw(FILE_MANIFEST_COPY);
####################################################################################################################################
# Default match factor
@@ -291,21 +295,23 @@ sub new
my
(
$strOperation,
- $strFileName, # Manifest filename
- $bLoad # Load the manifest?
+ $strFileName,
+ $bLoad,
+ $oStorage,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strFileName', trace => true},
- {name => 'bLoad', required => false, trace => true}
+ {name => 'bLoad', required => false, trace => true},
+ {name => 'oStorage', optional => true, default => storageRepo(), trace => true},
);
# Set defaults
$bLoad = defined($bLoad) ? $bLoad : true;
# Init object and store variables
- my $self = $class->SUPER::new($strFileName, {bLoad => $bLoad});
+ my $self = $class->SUPER::new($strFileName, {bLoad => $bLoad, oStorage => $oStorage});
# Return from function and log return values if any
return logDebugReturn
@@ -554,7 +560,7 @@ sub build
my
(
$strOperation,
- $oFile,
+ $oStorageDbMaster,
$strDbVersion,
$strPath,
$oLastManifest,
@@ -569,7 +575,7 @@ sub build
logDebugParam
(
__PACKAGE__ . '->build', \@_,
- {name => 'oFile'},
+ {name => 'oStorageDbMaster'},
{name => 'strDbVersion'},
{name => 'strPath'},
{name => 'oLastManifest', required => false},
@@ -589,7 +595,7 @@ sub build
# If not online then build the tablespace map from pg_tblspc path
if (!$bOnline && !defined($hTablespaceMap))
{
- my $hTablespaceManifest = $oFile->manifest(PATH_DB_ABSOLUTE, $strPath . '/' . DB_PATH_PGTBLSPC);
+ my $hTablespaceManifest = $oStorageDbMaster->manifest($strPath . '/' . DB_PATH_PGTBLSPC);
$hTablespaceMap = {};
foreach my $strOid (sort(CORE::keys(%{$hTablespaceManifest})))
@@ -632,11 +638,11 @@ sub build
confess &log(ASSERT, "cannot get manifest for '${strPath}' when no parent path is specified");
}
- $strPath = pathAbsolute($strParentPath, $strPath);
+ $strPath = $oStorageDbMaster->pathAbsolute($strParentPath, $strPath);
}
# Get the manifest for this level
- my $hManifest = $oFile->manifest(PATH_DB_ABSOLUTE, $strPath);
+ my $hManifest = $oStorageDbMaster->manifest($strPath);
my $strManifestType = MANIFEST_VALUE_LINK;
# Loop though all paths/files/links in the manifest
@@ -740,7 +746,7 @@ sub build
# Check for tablespaces in PGDATA
if (index($hManifest->{$strName}{link_destination}, "${strPath}/") == 0 ||
(index($hManifest->{$strName}{link_destination}, '/') != 0 &&
- index(pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC,
+ index($oStorageDbMaster->pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC,
$hManifest->{$strName}{link_destination}) . '/', "${strPath}/") == 0))
{
confess &log(ERROR, 'tablespace symlink ' . $hManifest->{$strName}{link_destination} .
@@ -805,7 +811,7 @@ sub build
$strPath = dirname("${strPath}/${strName}");
- $self->build($oFile, $strDbVersion, $strLinkDestination, undef, $bOnline, $hTablespaceMap, $hDatabaseMap,
+ $self->build($oStorageDbMaster, $strDbVersion, $strLinkDestination, undef, $bOnline, $hTablespaceMap, $hDatabaseMap,
$strFile, $bTablespace, $strPath, $strFilter, $strLinkDestination);
}
}
@@ -820,7 +826,9 @@ sub build
# lead to an invalid diff/incr backup later when using timestamps to determine which files have changed. Offline backups do
# not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be
# consistent anyway and the one-second resolution problem is the least of our worries).
- my $lTimeBegin = $oFile->wait(PATH_DB_ABSOLUTE, $bOnline);
+ my $lTimeBegin =
+ $oStorageDbMaster->can('protocol') ?
+ $oStorageDbMaster->protocol()->cmdExecute(OP_WAIT, [$bOnline]) : waitRemainder($bOnline);
# Check that links are valid
$self->linkCheck();
@@ -963,8 +971,8 @@ sub linkCheck
{
my $strChildPath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetChild, MANIFEST_SUBKEY_PATH);
- if (index(
- pathAbsolute($strBasePath, $strChildPath) . '/', pathAbsolute($strBasePath, $strParentPath) . '/') == 0)
+ if (index(storageLocal()->pathAbsolute(
+ $strBasePath, $strChildPath) . '/', storageLocal()->pathAbsolute($strBasePath, $strParentPath) . '/') == 0)
{
confess &log(ERROR, 'link ' . $self->dbPathGet($strBasePath, $strTargetChild) .
" (${strChildPath}) references a subdirectory of or" .
diff --git a/lib/pgBackRest/Protocol/Common/Master.pm b/lib/pgBackRest/Protocol/Base/Master.pm
similarity index 80%
rename from lib/pgBackRest/Protocol/Common/Master.pm
rename to lib/pgBackRest/Protocol/Base/Master.pm
index 182134f86..8ddff22a0 100644
--- a/lib/pgBackRest/Protocol/Common/Master.pm
+++ b/lib/pgBackRest/Protocol/Base/Master.pm
@@ -1,23 +1,31 @@
####################################################################################################################################
-# PROTOCOL COMMON MASTER MODULE
+# Protocol Master Base
####################################################################################################################################
-package pgBackRest::Protocol::Common::Master;
-use parent 'pgBackRest::Protocol::Common::Common';
+package pgBackRest::Protocol::Base::Master;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
-use File::Basename qw(dirname);
+use Exporter qw(import);
+ our @EXPORT = qw();
use Time::HiRes qw(gettimeofday);
+use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Version;
+####################################################################################################################################
+# Operation constants
+####################################################################################################################################
+use constant OP_NOOP => 'noop';
+ push @EXPORT, qw(OP_NOOP);
+use constant OP_EXIT => 'exit';
+ push @EXPORT, qw(OP_EXIT);
+
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
@@ -25,52 +33,35 @@ sub new
{
my $class = shift; # Class name
+ # Create the class hash
+ my $self = {};
+ bless $self, $class;
+
# Assign function parameters, defaults, and log debug info
- my
(
- $strOperation,
- $strRemoteType, # Type of remote (DB or BACKUP)
- $strName, # Name of the protocol
- $strId, # Id of this process for error messages
- $oIO, # IO object
- $iBufferMax, # Maximum buffer size
- $iCompressLevel, # Set compression level
- $iCompressLevelNetwork, # Set compression level for network only compression
- $iProtocolTimeout, # Protocol timeout
+ my $strOperation,
+ $self->{strName},
+ $self->{strId},
+ $self->{oIo},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
- {name => 'strRemoteType', trace => true},
{name => 'strName', trace => true},
{name => 'strId', trace => true},
- {name => 'strCommand', trace => true},
- {name => 'iBufferMax', trace => true},
- {name => 'iCompressLevel', trace => true},
- {name => 'iCompressLevelNetwork', trace => true},
- {name => 'iProtocolTimeout', trace => true},
+ {name => 'oIo', trace => true},
);
- # Create the class hash
- my $self = $class->SUPER::new($iBufferMax, $iCompressLevel, $iCompressLevelNetwork, $iProtocolTimeout, $strName);
- bless $self, $class;
-
- # Set remote to specified value
- $self->{strRemoteType} = $strRemoteType;
-
- # Set IO Object
- $self->{io} = $oIO;
+ # Create JSON object
+ $self->{oJSON} = JSON::PP->new()->allow_nonref();
# Check greeting to be sure the protocol matches
$self->greetingRead();
# Setup the keepalive timer
- $self->{fKeepAliveTimeout} = $iProtocolTimeout / 2 > 120 ? 120 : $iProtocolTimeout / 2;
+ $self->{fKeepAliveTimeout} = $self->io()->timeout() / 2 > 120 ? 120 : $self->io()->timeout() / 2;
$self->{fKeepAliveTime} = gettimeofday();
- # Set the id to be used for messages (especially error messages)
- $self->{strId} = $strId;
-
# Set the error prefix used when raising error messages
$self->{strErrorPrefix} = 'raised on ' . $self->{strId} . ' host';
@@ -102,9 +93,14 @@ sub greetingRead
my $self = shift;
# Get the first line of output from the remote if possible
- my $hGreeting = $self->{oJSON}->decode($self->{io}->lineRead(undef, undef, undef, true));
+ my $strGreeting = $self->io()->readLine(true);
+
+ # Check for errors
+ $self->io()->error();
# Error if greeting parameters do not match
+ my $hGreeting = $self->{oJSON}->decode($strGreeting);
+
for my $hParam ({strName => 'name', strExpected => BACKREST_NAME},
{strName => 'version', strExpected => BACKREST_VERSION},
{strName => 'service', strExpected => $self->{strName}})
@@ -117,8 +113,8 @@ sub greetingRead
}
}
- # Exchange one protocol message to catch errors early
- $self->outputRead();
+ # Perform noop to catch errors early
+ $self->noOp();
}
####################################################################################################################################
@@ -148,7 +144,7 @@ sub outputRead
{name => 'bRef', default => false, trace => true},
);
- my $strProtocolResult = $self->{io}->lineRead();
+ my $strProtocolResult = $self->io()->readLine();
logDebugMisc
(
@@ -179,7 +175,6 @@ sub outputRead
# If output is required and there is no output, raise exception
if ($bOutputRequired && !defined($hResult->{out}))
{
- $self->{io}->waitPid();
confess &log(ERROR, "$self->{strErrorPrefix}: output is not defined", ERROR_PROTOCOL_OUTPUT_REQUIRED);
}
@@ -223,7 +218,7 @@ sub cmdWrite
);
# Write out the command
- $self->{io}->lineWrite($strProtocolCommand);
+ $self->io()->writeLine($strProtocolCommand);
# Reset the keep alive time
$self->{fKeepAliveTime} = gettimeofday();
@@ -281,4 +276,10 @@ sub noOp
$self->{fKeepAliveTime} = gettimeofday();
}
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub io {shift->{oIo}}
+sub master {true}
+
1;
diff --git a/lib/pgBackRest/Protocol/Common/Minion.pm b/lib/pgBackRest/Protocol/Base/Minion.pm
similarity index 72%
rename from lib/pgBackRest/Protocol/Common/Minion.pm
rename to lib/pgBackRest/Protocol/Base/Minion.pm
index e79fb8a0e..c7b4fb8b9 100644
--- a/lib/pgBackRest/Protocol/Common/Minion.pm
+++ b/lib/pgBackRest/Protocol/Base/Minion.pm
@@ -1,8 +1,7 @@
####################################################################################################################################
-# PROTOCOL COMMON MINION MODULE
+# Protocol Minion Base
####################################################################################################################################
-package pgBackRest::Protocol::Common::Minion;
-use parent 'pgBackRest::Protocol::Common::Common';
+package pgBackRest::Protocol::Base::Minion;
use strict;
use warnings FATAL => qw(all);
@@ -15,7 +14,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Base::Master;
+use pgBackRest::Protocol::Helper;
use pgBackRest::Version;
####################################################################################################################################
@@ -25,45 +25,31 @@ sub new
{
my $class = shift; # Class name
+ # Create the class hash
+ my $self = {};
+ bless $self, $class;
+
# Assign function parameters, defaults, and log debug info
- my
(
- $strOperation,
- $strName, # Name of the protocol
- $strCommand, # Command the master process is running
- $oIO, # IO object
- $iBufferMax, # Maximum buffer size
- $iCompressLevel, # Set compression level
- $iCompressLevelNetwork, # Set compression level for network only compression
- $iProtocolTimeout, # Protocol timeout
+ my $strOperation,
+ $self->{strName},
+ $self->{oIo},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName', trace => true},
- {name => 'strCommand', trace => true},
- {name => 'oIO', trace => true},
- {name => 'iBufferMax', trace => true},
- {name => 'iCompressLevel', trace => true},
- {name => 'iCompressLevelNetwork', trace => true},
- {name => 'iProtocolTimeout', trace => true},
+ {name => 'oIo', trace => true},
);
- # Create the class hash
- my $self = $class->SUPER::new($iBufferMax, $iCompressLevel, $iCompressLevelNetwork, $iProtocolTimeout, $strName);
- bless $self, $class;
-
- # Set IO Object
- $self->{io} = $oIO;
-
- # Set command the master is running
- $self->{strCommand} = $strCommand;
+ # Create JSON object
+ $self->{oJSON} = JSON::PP->new()->allow_nonref();
# Write the greeting so master process knows who we are
$self->greetingWrite();
# Initialize module variables
- $self->{hCommandMap} = $self->init();
+ $self->{hCommandMap} = $self->can('init') ? $self->init() : undef;
# Return from function and log return values if any
return logDebugReturn
@@ -83,24 +69,8 @@ sub greetingWrite
my $self = shift;
# Write the greeting
- $self->{io}->lineWrite((JSON::PP->new()->canonical()->allow_nonref())->encode(
+ $self->io()->writeLine((JSON::PP->new()->canonical()->allow_nonref())->encode(
{name => BACKREST_NAME, service => $self->{strName}, version => BACKREST_VERSION}));
-
- # Exchange one protocol message to catch errors early
- $self->outputWrite();
-}
-
-####################################################################################################################################
-# binaryXferAbort
-#
-# Abort transfer when source file does not exist.
-####################################################################################################################################
-sub binaryXferAbort
-{
- my $self = shift;
-
- # Only allow in the backend process
- $self->{io}->lineWrite('block -1');
}
####################################################################################################################################
@@ -120,7 +90,7 @@ sub errorWrite
}
# Write error code and message
- $self->{io}->lineWrite($self->{oJSON}->encode({err => $oException->code(), out => $oException->message()}));
+ $self->io()->writeLine($self->{oJSON}->encode({err => $oException->code(), out => $oException->message()}));
}
####################################################################################################################################
@@ -132,7 +102,7 @@ sub outputWrite
{
my $self = shift;
- $self->{io}->lineWrite($self->{oJSON}->encode({out => \@_}));
+ $self->io()->writeLine($self->{oJSON}->encode({out => \@_}));
}
####################################################################################################################################
@@ -144,7 +114,7 @@ sub cmdRead
{
my $self = shift;
- my $hCommand = $self->{oJSON}->decode($self->{io}->lineRead());
+ my $hCommand = $self->{oJSON}->decode($self->io()->readLine());
return $hCommand->{cmd}, $hCommand->{param};
}
@@ -178,6 +148,7 @@ sub process
# Run the standard NOOP command. This this can be overridden in hCommandMap to implement a custom NOOP.
elsif ($strCommand eq OP_NOOP)
{
+ protocolKeepAlive();
$self->outputWrite();
}
else
@@ -185,12 +156,6 @@ sub process
confess "invalid command: ${strCommand}";
}
- # Run the post command if defined
- if (defined($self->{hCommandMap}{&OP_POST}))
- {
- $self->{hCommandMap}{&OP_POST}->($strCommand, $rParam);
- }
-
return true;
}
# Process errors
@@ -222,4 +187,10 @@ sub process
return 0;
}
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub io {shift->{oIo}}
+sub master {false}
+
1;
diff --git a/lib/pgBackRest/Protocol/Command/Master.pm b/lib/pgBackRest/Protocol/Command/Master.pm
index ead69d2ba..4afa792f6 100644
--- a/lib/pgBackRest/Protocol/Command/Master.pm
+++ b/lib/pgBackRest/Protocol/Command/Master.pm
@@ -2,7 +2,7 @@
# PROTOCOL COMMAND MASTER MODULE
####################################################################################################################################
package pgBackRest::Protocol::Command::Master;
-use parent 'pgBackRest::Protocol::Common::Master';
+use parent 'pgBackRest::Protocol::Base::Master';
use strict;
use warnings FATAL => qw(all);
@@ -15,8 +15,8 @@ use Time::HiRes qw(gettimeofday);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
-use pgBackRest::Protocol::Common::Common;
-use pgBackRest::Protocol::Common::Io::Process;
+use pgBackRest::Common::Io::Process;
+use pgBackRest::Protocol::Base::Master;
use pgBackRest::Version;
####################################################################################################################################
@@ -30,7 +30,6 @@ sub new
my
(
$strOperation,
- $strRemoteType, # Type of remote (DB or BACKUP)
$strName, # Name of the protocol
$strId, # Id of this process for error messages
$strCommand, # Command to execute on local/remote
@@ -42,7 +41,6 @@ sub new
logDebugParam
(
__PACKAGE__ . '->new', \@_,
- {name => 'strRemoteType'},
{name => 'strName'},
{name => 'strId'},
{name => 'strCommand'},
@@ -59,11 +57,12 @@ sub new
}
# Execute the command
- my $oIO = pgBackRest::Protocol::Common::Io::Process->new3($strId, $strCommand, $iProtocolTimeout, $iBufferMax);
+ my $oIo = new pgBackRest::Common::Io::Process(
+ new pgBackRest::Common::Io::Buffered(
+ new pgBackRest::Common::Io::Handle($strId), $iProtocolTimeout, $iBufferMax), $strCommand);
# Create the class hash
- my $self = $class->SUPER::new(
- $strRemoteType, $strName, $strId, $oIO, $iBufferMax, $iCompressLevel, $iCompressLevelNetwork, $iProtocolTimeout);
+ my $self = $class->SUPER::new($strName, $strId, $oIo);
bless $self, $class;
# Return from function and log return values if any
@@ -98,13 +97,13 @@ sub close
my $bClosed = false;
# Only send the exit command if the process is running
- if (defined($self->{io}) && defined($self->{io}->processId()))
+ if (defined($self->io()) && defined($self->io()->processId()))
{
&log(TRACE, "sending exit command to process");
eval
{
- $self->cmdWrite('exit');
+ $self->cmdWrite(OP_EXIT);
return true;
}
or do
@@ -133,7 +132,8 @@ sub close
($bComplete ? "\n${strHint}" : ''));
};
- undef($self->{io});
+ $self->{oIo}->close();
+ undef($self->{oIo});
$bClosed = true;
}
diff --git a/lib/pgBackRest/Protocol/Command/Minion.pm b/lib/pgBackRest/Protocol/Command/Minion.pm
index af258ce76..8618a0ad2 100644
--- a/lib/pgBackRest/Protocol/Command/Minion.pm
+++ b/lib/pgBackRest/Protocol/Command/Minion.pm
@@ -2,7 +2,7 @@
# PROTOCOL COMMAND MINION MODULE
####################################################################################################################################
package pgBackRest::Protocol::Command::Minion;
-use parent 'pgBackRest::Protocol::Common::Minion';
+use parent 'pgBackRest::Protocol::Base::Minion';
use strict;
use warnings FATAL => qw(all);
@@ -15,8 +15,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
-use pgBackRest::Protocol::Common::Minion;
-use pgBackRest::Protocol::Common::Io::Process;
+use pgBackRest::Protocol::Base::Minion;
+use pgBackRest::Common::Io::Buffered;
use pgBackRest::Version;
####################################################################################################################################
@@ -31,28 +31,24 @@ sub new
(
$strOperation,
$strName, # Name of the protocol
- $strCommand, # Command the master process is running
$iBufferMax, # Maximum buffer size
- $iCompressLevel, # Set compression level
- $iCompressLevelNetwork, # Set compression level for network only compression
$iProtocolTimeout, # Protocol timeout
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName'},
- {name => 'strCommand'},
{name => 'iBufferMax'},
- {name => 'iCompressLevel'},
- {name => 'iCompressLevelNetwork'},
{name => 'iProtocolTimeout'},
);
+ # Open buffered protocol io
+ my $oIo =
+ new pgBackRest::Common::Io::Buffered(
+ new pgBackRest::Common::Io::Handle('stdio', *STDIN, *STDOUT), $iProtocolTimeout, $iBufferMax);
+
# Create the class hash
- my $self = $class->SUPER::new(
- $strName, $strCommand,
- new pgBackRest::Protocol::Common::Io::Process(*STDIN, *STDOUT, *STDERR, undef, undef, $iProtocolTimeout, $iBufferMax),
- $iBufferMax, $iCompressLevel, $iCompressLevelNetwork, $iProtocolTimeout);
+ my $self = $class->SUPER::new($strName, $oIo);
bless $self, $class;
# Return from function and log return values if any
diff --git a/lib/pgBackRest/Protocol/Common/Common.pm b/lib/pgBackRest/Protocol/Common/Common.pm
deleted file mode 100644
index 654e85771..000000000
--- a/lib/pgBackRest/Protocol/Common/Common.pm
+++ /dev/null
@@ -1,735 +0,0 @@
-####################################################################################################################################
-# PROTOCOL COMMON MODULE
-####################################################################################################################################
-package pgBackRest::Protocol::Common::Common;
-
-use strict;
-use warnings FATAL => qw(all);
-use Carp qw(confess);
-
-use Exporter qw(import);
- our @EXPORT = qw();
-use Compress::Raw::Zlib qw(WANT_GZIP Z_OK Z_BUF_ERROR Z_STREAM_END);
-use File::Basename qw(dirname);
-use JSON::PP;
-
-use pgBackRest::Common::Exception;
-use pgBackRest::Common::Ini;
-use pgBackRest::Common::Log;
-use pgBackRest::Protocol::Common::Io::Process;
-
-####################################################################################################################################
-# DB/BACKUP Constants
-####################################################################################################################################
-use constant DB => 'db';
- push @EXPORT, qw(DB);
-use constant BACKUP => 'backup';
- push @EXPORT, qw(BACKUP);
-use constant NONE => 'none';
- push @EXPORT, qw(NONE);
-
-####################################################################################################################################
-# Operation constants
-####################################################################################################################################
-use constant OP_NOOP => 'noop';
- push @EXPORT, qw(OP_NOOP);
-use constant OP_EXIT => 'exit';
- push @EXPORT, qw(OP_EXIT);
-
-# Backup module
-use constant OP_BACKUP_FILE => 'backupFile';
- push @EXPORT, qw(OP_BACKUP_FILE);
-
-# Archive Module
-use constant OP_ARCHIVE_GET_ARCHIVE_ID => 'archiveId';
- push @EXPORT, qw(OP_ARCHIVE_GET_ARCHIVE_ID);
-use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';
- push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);
-use constant OP_ARCHIVE_PUSH_CHECK => 'archivePushCheck';
- push @EXPORT, qw(OP_ARCHIVE_PUSH_CHECK);
-
-# Archive Push Async Module
-use constant OP_ARCHIVE_PUSH_ASYNC => 'archivePushAsync';
- push @EXPORT, qw(OP_ARCHIVE_PUSH_ASYNC);
-
-# Archive File Module
-use constant OP_ARCHIVE_PUSH_FILE => 'archivePushFile';
- push @EXPORT, qw(OP_ARCHIVE_PUSH_FILE);
-
-# Check Module
-use constant OP_CHECK_BACKUP_INFO_CHECK => 'backupInfoCheck';
- push @EXPORT, qw(OP_CHECK_BACKUP_INFO_CHECK);
-
-# Db Module
-use constant OP_DB_CONNECT => 'dbConnect';
- push @EXPORT, qw(OP_DB_CONNECT);
-use constant OP_DB_EXECUTE_SQL => 'dbExecSql';
- push @EXPORT, qw(OP_DB_EXECUTE_SQL);
-use constant OP_DB_INFO => 'dbInfo';
- push @EXPORT, qw(OP_DB_INFO);
-
-# File Module
-use constant OP_FILE_COPY => 'fileCopy';
- push @EXPORT, qw(OP_FILE_COPY);
-use constant OP_FILE_COPY_IN => 'fileCopyIn';
- push @EXPORT, qw(OP_FILE_COPY_IN);
-use constant OP_FILE_COPY_OUT => 'fileCopyOut';
- push @EXPORT, qw(OP_FILE_COPY_OUT);
-use constant OP_FILE_EXISTS => 'fileExists';
- push @EXPORT, qw(OP_FILE_EXISTS);
-use constant OP_FILE_LIST => 'fileList';
- push @EXPORT, qw(OP_FILE_LIST);
-use constant OP_FILE_MANIFEST => 'fileManifest';
- push @EXPORT, qw(OP_FILE_MANIFEST);
-use constant OP_FILE_PATH_CREATE => 'pathCreate';
- push @EXPORT, qw(OP_FILE_PATH_CREATE);
-use constant OP_FILE_WAIT => 'wait';
- push @EXPORT, qw(OP_FILE_WAIT);
-
-# Info module
-use constant OP_INFO_STANZA_LIST => 'infoStanzList';
- push @EXPORT, qw(OP_INFO_STANZA_LIST);
-
-# Restore module
-use constant OP_RESTORE_FILE => 'restoreFile';
- push @EXPORT, qw(OP_RESTORE_FILE);
-
-# To be run after each command
-use constant OP_POST => 'post';
- push @EXPORT, qw(OP_POST);
-
-####################################################################################################################################
-# CONSTRUCTOR
-####################################################################################################################################
-sub new
-{
- my $class = shift; # Class name
-
- # Create the class hash
- my $self = {};
- bless $self, $class;
-
- # Assign function parameters, defaults, and log debug info
- (
- my $strOperation,
- $self->{iBufferMax},
- $self->{iCompressLevel},
- $self->{iCompressLevelNetwork},
- $self->{iProtocolTimeout},
- $self->{strName}
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new', \@_,
- {name => 'iBufferMax', trace => true},
- {name => 'iCompressLevel', trace => true},
- {name => 'iCompressNetworkLevel', trace => true},
- {name => 'iProtocolTimeout', trace => true},
- {name => 'strName', required => false, trace => true}
- );
-
- # By default remote type is NONE
- $self->{strRemoteType} = NONE;
-
- # Create JSON object
- $self->{oJSON} = JSON::PP->new()->allow_nonref();
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'self', value => $self}
- );
-}
-
-####################################################################################################################################
-# keepAlive
-#
-# Don't do anything for keep alive if there is no remote.
-####################################################################################################################################
-sub keepAlive
-{
-}
-
-####################################################################################################################################
-# noop
-#
-# Don't do anything for noop if there is no remote.
-####################################################################################################################################
-sub noOp
-{
-}
-
-####################################################################################################################################
-# blockRead
-#
-# Read a block from the protocol layer.
-####################################################################################################################################
-sub blockRead
-{
- my $self = shift;
- my $oIn = shift;
- my $strBlockRef = shift;
- my $bProtocol = shift;
-
- my $iBlockSize;
- my $hMessage;
-
- if ($bProtocol)
- {
- # Read the block header and make sure it's valid
- my $strBlockHeader = $oIn->lineRead();
-
- if ($strBlockHeader !~ /^block -{0,1}[0-9]+( .*){0,1}$/)
- {
- confess &log(ERROR, "invalid block header ${strBlockHeader}", ERROR_FILE_READ);
- }
-
- # Get block size from the header
- my @stryToken = split(/ /, $strBlockHeader);
- $iBlockSize = $stryToken[1];
-
- if (defined($stryToken[2]))
- {
- $hMessage = $self->{oJSON}->decode($stryToken[2]);
- }
-
- # If block size is 0 or an error code then undef the buffer
- if ($iBlockSize <= 0)
- {
- undef($$strBlockRef);
- }
- # Else read the block
- else
- {
- $oIn->bufferRead($strBlockRef, $iBlockSize, undef, true);
- }
- }
- else
- {
- $iBlockSize = $oIn->bufferRead($strBlockRef, $self->{iBufferMax}, defined($$strBlockRef) ? length($$strBlockRef) : undef);
- }
-
- # Return the block size
- return $iBlockSize, $hMessage;
-}
-
-####################################################################################################################################
-# blockWrite
-#
-# Write a block to the protocol layer.
-####################################################################################################################################
-sub blockWrite
-{
- my $self = shift;
- my $oOut = shift;
- my $tBlockRef = shift;
- my $iBlockSize = shift;
- my $bProtocol = shift;
- my $hMessage = shift;
-
- # If block size is not defined, get it from buffer length
- $iBlockSize = defined($iBlockSize) ? $iBlockSize : length($$tBlockRef);
-
- # Write block header to the protocol stream
- if ($bProtocol)
- {
- $oOut->lineWrite("block ${iBlockSize}" . (defined($hMessage) ? " " . $self->{oJSON}->encode($hMessage) : ''));
- }
-
- # Write block if size > 0
- if ($iBlockSize > 0)
- {
- $oOut->bufferWrite($tBlockRef, $iBlockSize);
- }
-}
-
-####################################################################################################################################
-# binaryXfer
-#
-# Copies data from one file handle to another, optionally compressing or decompressing the data in stream. If $strRemote != none
-# then one side is a protocol stream, though this can be controlled with the bProtocol param.
-####################################################################################################################################
-sub binaryXfer
-{
- my $self = shift;
- my $hIn = shift;
- my $hOut = shift;
- my $strRemote = shift;
- my $bSourceCompressed = shift;
- my $bDestinationCompress = shift;
- my $bProtocol = shift;
- my $fnExtra = shift;
- my $rExtraParam = shift;
-
- # The input stream must be defined
- my $oIn;
-
- if (!defined($hIn))
- {
- $oIn = $self->{io};
- }
- else
- {
- $oIn = new pgBackRest::Protocol::Common::Io::Process(
- $hIn, undef, $self->{io}->{hErr}, $self->{io}->{pid}, $self->{io}->{strId}, $self->{iProtocolTimeout},
- $self->{iBufferMax});
- }
-
- # The output stream must be defined unless 'none' is passed
- my $oOut;
-
- if (!defined($hOut))
- {
- $oOut = $self->{io};
- }
- elsif ($hOut ne 'none')
- {
- $oOut = new pgBackRest::Protocol::Common::Io::Process(
- undef, $hOut, $self->{io}->{hErr}, $self->{io}->{pid}, $self->{io}->{strId}, $self->{iProtocolTimeout},
- $self->{iBufferMax});
- }
-
- # If no remote is defined then set to none
- if (!defined($strRemote))
- {
- $strRemote = 'none';
- }
- # Only set compression defaults when remote is defined
- else
- {
- $bSourceCompressed = defined($bSourceCompressed) ? $bSourceCompressed : false;
- $bDestinationCompress = defined($bDestinationCompress) ? $bDestinationCompress : false;
- }
-
- # Default protocol to true
- $bProtocol = defined($bProtocol) ? $bProtocol : true;
- my $hMessage = undef;
-
- # Checksum, size, and extra
- my $strChecksum = undef;
- my $iFileSize = undef;
- my $rExtra = undef;
-
- # Read from the protocol stream
- if ($strRemote eq 'in')
- {
- # If the destination should not be compressed then decompress
- if (!$bDestinationCompress)
- {
- my $iBlockSize;
- my $tCompressedBuffer;
- my $tUncompressedBuffer;
- my $iUncompressedBufferSize;
-
- # Initialize SHA
- my $oSHA;
-
- if (!$bProtocol)
- {
- $oSHA = Digest::SHA->new('sha1');
- }
-
- # Initialize inflate object and check for errors
- my ($oZLib, $iZLibStatus) =
- new Compress::Raw::Zlib::Inflate(WindowBits => 15 & $bSourceCompressed ? WANT_GZIP : 0,
- Bufsize => $self->{iBufferMax}, LimitOutput => 1);
-
- if ($iZLibStatus != Z_OK)
- {
- confess &log(ERROR, "unable create a inflate object: ${iZLibStatus}");
- }
-
- # Read all input
- do
- {
- # Read a block from the input stream
- ($iBlockSize, $hMessage) = $self->blockRead($oIn, \$tCompressedBuffer, $bProtocol);
-
- # Process protocol messages
- if (defined($hMessage) && defined($hMessage->{bChecksum}) && !$hMessage->{bChecksum})
- {
- $oSHA = Digest::SHA->new('sha1');
- undef($hMessage);
- }
-
- # If the block contains data, decompress it
- if ($iBlockSize > 0)
- {
- # Keep looping while there is more to decompress
- do
- {
- # Decompress data
- $iZLibStatus = $oZLib->inflate($tCompressedBuffer, $tUncompressedBuffer);
- $iUncompressedBufferSize = length($tUncompressedBuffer);
-
- # If status is ok, write the data
- if ($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR || $iZLibStatus == Z_STREAM_END)
- {
- if ($iUncompressedBufferSize > 0)
- {
- # Add data to checksum
- if (defined($oSHA))
- {
- $oSHA->add($tUncompressedBuffer);
- }
-
- # Write data if hOut is defined
- if (defined($oOut))
- {
- $oOut->bufferWrite(\$tUncompressedBuffer, $iUncompressedBufferSize);
- }
- }
- }
- # Else error, exit so it can be handled
- else
- {
- $iBlockSize = 0;
- }
- }
- while ($iZLibStatus != Z_STREAM_END && $iUncompressedBufferSize > 0 && $iBlockSize > 0);
- }
- }
- while ($iBlockSize > 0);
-
- # Make sure the decompression succeeded (iBlockSize < 0 indicates remote error, handled later)
- if ($iBlockSize == 0 && $iZLibStatus != Z_STREAM_END)
- {
- confess &log(ERROR, "unable to inflate stream: ${iZLibStatus}");
- }
-
- # Get checksum and total uncompressed bytes written
- if (defined($oSHA))
- {
- $strChecksum = $oSHA->hexdigest();
- $iFileSize = $oZLib->total_out();
- };
- }
- # If the destination should be compressed then just write out the already compressed stream
- else
- {
- my $iBlockSize;
- my $tBuffer;
-
- # Initialize checksum and size
- my $oSHA;
-
- if (!$bProtocol)
- {
- $oSHA = Digest::SHA->new('sha1');
- $iFileSize = 0;
- $rExtra = defined($fnExtra) ? {} : undef;
- }
-
- do
- {
- # Read a block from the protocol stream
- ($iBlockSize, $hMessage) = $self->blockRead($oIn, \$tBuffer, $bProtocol);
-
- # Add data to checksum and size
- if ($iBlockSize > 0 && !$bProtocol)
- {
- $oSHA->add($tBuffer);
- $iFileSize += $iBlockSize;
- }
-
- # Do extra processing on the buffer if requested
- if (!$bProtocol && defined($fnExtra))
- {
- $fnExtra->($rExtraParam, \$tBuffer, $iBlockSize, $iFileSize - $iBlockSize, $rExtra);
- }
-
- # Write buffer
- if ($iBlockSize > 0)
- {
- $oOut->bufferWrite(\$tBuffer, $iBlockSize);
- undef($tBuffer);
- }
- }
- while ($iBlockSize > 0);
-
- # Get checksum
- if (!$bProtocol)
- {
- $strChecksum = $oSHA->hexdigest();
- };
- }
- }
- # Read from file input stream
- else
- {
- # If source is not already compressed then compress it
- if ($strRemote eq 'out' && !$bSourceCompressed)
- {
- my $iBlockSize;
- my $tCompressedBuffer;
- my $iCompressedBufferSize;
- my $tUncompressedBuffer;
-
- # Initialize message to indicate that a checksum will be sent
- if ($bProtocol && defined($oOut))
- {
- $hMessage->{bChecksum} = true;
- }
-
- # Initialize checksum
- my $oSHA = Digest::SHA->new('sha1');
- $rExtra = defined($fnExtra) ? {} : undef;
-
- # Initialize inflate object and check for errors
- my ($oZLib, $iZLibStatus) =
- new Compress::Raw::Zlib::Deflate(WindowBits => 15 & $bDestinationCompress ? WANT_GZIP : 0,
- Level => $bDestinationCompress ? $self->{iCompressLevel} :
- $self->{iCompressLevelNetwork},
- Bufsize => $self->{iBufferMax}, AppendOutput => 1);
-
- if ($iZLibStatus != Z_OK)
- {
- confess &log(ERROR, "unable create a deflate object: ${iZLibStatus}");
- }
-
- do
- {
- # Read a block from the stream
- $iBlockSize = $oIn->bufferRead(\$tUncompressedBuffer, $self->{iBufferMax});
-
- # If block size > 0 then update checksum and size
- if ($iBlockSize > 0)
- {
- # Update checksum and filesize
- $oSHA->add($tUncompressedBuffer);
- }
-
- # Do extra processing on the buffer if requested
- if (defined($fnExtra))
- {
- $fnExtra->($rExtraParam, \$tUncompressedBuffer, $iBlockSize, $oZLib->total_in(), $rExtra);
- }
-
- # If block size > 0 then compress
- if ($iBlockSize > 0)
- {
- # Compress the data
- $iZLibStatus = $oZLib->deflate($tUncompressedBuffer, $tCompressedBuffer);
- $iCompressedBufferSize = length($tCompressedBuffer);
-
- # If compression was successful
- if ($iZLibStatus == Z_OK)
- {
- # The compressed data is larger than block size, then write
- if ($iCompressedBufferSize > $self->{iBufferMax})
- {
- $self->blockWrite($oOut, \$tCompressedBuffer, $iCompressedBufferSize, $bProtocol, $hMessage);
- undef($tCompressedBuffer);
- undef($hMessage);
- }
- }
- # Else if error
- else
- {
- $iBlockSize = 0;
- }
- }
- }
- while ($iBlockSize > 0);
-
- # If good so far flush out the last bytes
- if ($iZLibStatus == Z_OK)
- {
- $iZLibStatus = $oZLib->flush($tCompressedBuffer);
- }
-
- # Make sure the compression succeeded
- if ($iZLibStatus != Z_OK)
- {
- confess &log(ERROR, "unable to deflate stream: ${iZLibStatus}");
- }
-
- # Get checksum and total uncompressed bytes written
- $strChecksum = $oSHA->hexdigest();
- $iFileSize = $oZLib->total_in();
-
- # Write out the last block
- if (defined($oOut))
- {
- $iCompressedBufferSize = length($tCompressedBuffer);
-
- if ($iCompressedBufferSize > 0)
- {
- $self->blockWrite($oOut, \$tCompressedBuffer, $iCompressedBufferSize, $bProtocol, $hMessage);
- undef($hMessage);
- }
-
- $self->blockWrite(
- $oOut, undef, 0, $bProtocol, {strChecksum => $strChecksum, iFileSize => $iFileSize, rExtra => $rExtra});
- }
- }
- # If source is already compressed or transfer is not compressed then just read the stream
- else
- {
- my $iBlockSize;
- my $tBuffer;
- my $tCompressedBuffer;
- my $tUncompressedBuffer;
- my $iUncompressedBufferSize;
- my $oSHA;
- my $oZLib;
- my $iZLibStatus;
-
- # If the destination will be compressed setup deflate
- if ($bDestinationCompress)
- {
- if ($bProtocol)
- {
- $hMessage->{bChecksum} = true;
- }
-
- # Initialize checksum and size
- $oSHA = Digest::SHA->new('sha1');
- $iFileSize = 0;
-
- # Initialize inflate object and check for errors
- ($oZLib, $iZLibStatus) =
- new Compress::Raw::Zlib::Inflate(WindowBits => WANT_GZIP, Bufsize => $self->{iBufferMax}, LimitOutput => 1);
-
- if ($iZLibStatus != Z_OK)
- {
- confess &log(ERROR, "unable create a inflate object: ${iZLibStatus}");
- }
- }
- # Initialize message to indicate that a checksum will not be sent
- elsif ($bProtocol)
- {
- $hMessage->{bChecksum} = false;
- }
-
- # Read input
- do
- {
- $iBlockSize = $oIn->bufferRead(\$tBuffer, $self->{iBufferMax});
-
- # Write a block if size > 0
- if ($iBlockSize > 0)
- {
- $self->blockWrite($oOut, \$tBuffer, $iBlockSize, $bProtocol, $hMessage);
- undef($hMessage);
- }
-
- # Decompress the buffer to calculate checksum/size
- if ($bDestinationCompress)
- {
- # If the block contains data, decompress it
- if ($iBlockSize > 0)
- {
- # Copy file buffer to compressed buffer
- if (defined($tCompressedBuffer))
- {
- $tCompressedBuffer .= $tBuffer;
- }
- else
- {
- $tCompressedBuffer = $tBuffer;
- }
-
- # Keep looping while there is more to decompress
- do
- {
- # Decompress data
- $iZLibStatus = $oZLib->inflate($tCompressedBuffer, $tUncompressedBuffer);
- $iUncompressedBufferSize = length($tUncompressedBuffer);
-
- # If status is ok, write the data
- if ($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR || $iZLibStatus == Z_STREAM_END)
- {
- if ($iUncompressedBufferSize > 0)
- {
- $oSHA->add($tUncompressedBuffer);
- $iFileSize += $iUncompressedBufferSize;
- }
- }
- # Else error, exit so it can be handled
- else
- {
- $iBlockSize = 0;
- }
- }
- while ($iZLibStatus != Z_STREAM_END && $iUncompressedBufferSize > 0 && $iBlockSize > 0);
- }
- }
- }
- while ($iBlockSize > 0);
-
- # Check decompression, get checksum
- if ($bDestinationCompress)
- {
- # Make sure the decompression succeeded (iBlockSize < 0 indicates remote error, handled later)
- if ($iBlockSize == 0 && $iZLibStatus != Z_STREAM_END)
- {
- confess &log(ERROR, "unable to inflate stream: ${iZLibStatus}");
- }
-
- # If protocol then create the message
- if ($bProtocol)
- {
- $hMessage = {strChecksum => $oSHA->hexdigest(), iFileSize => $iFileSize, rExtra => $rExtra};
- }
- # Otherwise just set checksum
- else
- {
- $strChecksum = $oSHA->hexdigest();
- }
- }
-
- # If protocol write
- if ($bProtocol)
- {
- # Write 0 block to indicate end of stream
- $self->blockWrite($oOut, undef, 0, $bProtocol, $hMessage);
- }
- }
- }
-
- # If message is defined then the checksum, size, and extra should be in it
- if (defined($hMessage))
- {
- return $hMessage->{strChecksum}, $hMessage->{iFileSize}, $hMessage->{rExtra};
- }
-
- # Return the checksum and size if they are available
- return $strChecksum, $iFileSize, $rExtra;
-}
-
-####################################################################################################################################
-# remoteType
-#
-# Return the remote type.
-####################################################################################################################################
-sub remoteType
-{
- return shift->{strRemoteType};
-}
-
-####################################################################################################################################
-# remoteTypeTest
-#
-# Determine if the remote matches the specified remote.
-####################################################################################################################################
-sub remoteTypeTest
-{
- my $self = shift;
- my $strRemoteType = shift;
-
- return $self->remoteType() eq $strRemoteType ? true : false;
-}
-
-####################################################################################################################################
-# isRemote
-#
-# Determine if the protocol object is communicating with a remote.
-####################################################################################################################################
-sub isRemote
-{
- return shift->{strRemoteType} ne NONE ? true : false;
-}
-
-1;
diff --git a/lib/pgBackRest/Protocol/Common/Io/Handle.pm b/lib/pgBackRest/Protocol/Common/Io/Handle.pm
deleted file mode 100644
index f1a48bd87..000000000
--- a/lib/pgBackRest/Protocol/Common/Io/Handle.pm
+++ /dev/null
@@ -1,462 +0,0 @@
-####################################################################################################################################
-# PROTOCOL IO MODULE
-####################################################################################################################################
-package pgBackRest::Protocol::Common::Io::Handle;
-
-use strict;
-use warnings FATAL => qw(all);
-use Carp qw(confess);
-use English '-no_match_vars';
-
-use Exporter qw(import);
- our @EXPORT = qw();
-use File::Basename qw(dirname);
-use IPC::Open3 qw(open3);
-use IO::Select;
-use POSIX qw(:sys_wait_h);
-use Symbol 'gensym';
-use Time::HiRes qw(gettimeofday);
-
-use pgBackRest::Common::Exception;
-use pgBackRest::Common::Log;
-use pgBackRest::Common::String;
-use pgBackRest::Common::Wait;
-
-####################################################################################################################################
-# Amount of time to attempt to retrieve errors when a process terminates unexpectedly
-####################################################################################################################################
-use constant IO_ERROR_TIMEOUT => 5;
- push @EXPORT, qw(IO_ERROR_TIMEOUT);
-
-####################################################################################################################################
-# CONSTRUCTOR
-####################################################################################################################################
-sub new
-{
- my $class = shift;
-
- # Create the class hash
- my $self = {};
- bless $self, $class;
-
- # Assign function parameters, defaults, and log debug info
- (
- my $strOperation,
- $self->{hIn}, # Input stream
- $self->{hOut}, # Output stream
- $self->{hErr}, # Error stream
- $self->{iProtocolTimeout}, # Protocol timeout
- $self->{iBufferMax}, # Maximum buffer size
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new', \@_,
- {name => 'hIn', required => false, trace => true},
- {name => 'hOut', required => false, trace => true},
- {name => 'hErr', required => false, trace => true},
- {name => 'iProtocolTimeout', trace => true},
- {name => 'iBufferMax', trace => true}
- );
-
- if (defined($self->{hIn}))
- {
- $self->{oInSelect} = IO::Select->new();
- $self->{oInSelect}->add($self->{hIn});
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'self', value => $self}
- );
-}
-
-####################################################################################################################################
-# new3
-#
-# Use open3 to run the command and get the io handles.
-####################################################################################################################################
-sub new3
-{
- my $class = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strId,
- $strCommand,
- $iProtocolTimeout, # Protocol timeout
- $iBufferMax # Maximum buffer Size
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new3', \@_,
- {name => 'strId', trace => true},
- {name => 'strCommand', trace => true},
- {name => 'iProtocolTimeout', trace => true},
- {name => 'iBufferMax', trace => true}
- );
-
- # Use open3 to run the command
- my ($pId, $hIn, $hOut, $hErr);
- $hErr = gensym;
-
- $pId = IPC::Open3::open3($hIn, $hOut, $hErr, $strCommand);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'self', value => $class->new($hOut, $hIn, $hErr, $pId, $strId, $iProtocolTimeout, $iBufferMax)}
- );
-}
-
-####################################################################################################################################
-# Get input/output/error handles
-####################################################################################################################################
-sub inputHandle
-{
- my $self = shift;
-
- return $self->{hIn};
-}
-
-sub outputHandle
-{
- my $self = shift;
-
- return $self->{hOut};
-}
-
-sub errorHandle
-{
- my $self = shift;
-
- return $self->{hErr};
-}
-
-####################################################################################################################################
-# lineRead
-#
-# Read an lf-terminated line from the input or error stream.
-####################################################################################################################################
-sub lineRead
-{
- my $self = shift;
- my $iTimeout = shift;
- my $bInRead = shift;
- my $bError = shift;
- my $bIgnoreEOF = shift;
-
- # If there's already data in the buffer try to find the next linefeed
- my $iLineFeedPos = defined($self->{strBuffer}) ? index($self->{strBuffer}, "\n", $self->{iBufferPos}) : -1;
-
- # Store the current time if a timeout is required
- if ($iLineFeedPos == -1)
- {
- my $fTimeout = defined($iTimeout) ? $iTimeout : $self->{iProtocolTimeout};
- my $fRemaining = $fTimeout;
- my $fTimeStart = gettimeofday();
-
- # If no linefeed was found then load more data
- do
- {
- # If the buffer already has data
- if (defined($self->{strBuffer}) && $self->{iBufferPos} < $self->{iBufferSize})
- {
- # And the buffer position is not 0 then trim it so there's room for more data
- if ($self->{iBufferPos} != 0)
- {
- $self->{strBuffer} = substr($self->{strBuffer}, $self->{iBufferPos});
- $self->{iBufferSize} = $self->{iBufferSize} - $self->{iBufferPos};
- $self->{iBufferPos} = 0;
- }
- }
- # Else the buffer is empty and data will need to be loaded
- else
- {
- undef($self->{strBuffer}); # ??? Do we need this?
- $self->{iBufferSize} = 0;
- $self->{iBufferPos} = 0;
- }
-
- # Get stream handle and select object
- my $hIn;
- my $oSelect;
-
- # If this is a normal input read
- if (!defined($bInRead) || $bInRead)
- {
- $hIn = $self->{hIn};
- $oSelect = $self->{oInSelect};
- }
- # Else this is an error read
- else
- {
- # The select object will need to be created the first time an error is read
- if (!defined($self->{oErrSelect}))
- {
- $self->{oErrSelect} = IO::Select->new();
- $self->{oErrSelect}->add($self->{hErr});
- }
-
- $oSelect = $self->{oErrSelect};
- $hIn = $self->{hErr};
- }
-
- # Load data into the buffer
- my $iBufferRead = 0;
-
- if ($oSelect->can_read($fRemaining))
- {
- $iBufferRead = sysread($hIn, $self->{strBuffer}, $self->{iBufferSize} >= $self->{iBufferMax} ?
- $self->{iBufferMax} : $self->{iBufferMax} - $self->{iBufferSize}, $self->{iBufferSize});
-
- # An error occurred if undef is returned
- if (!defined($iBufferRead))
- {
- # Store the error message
- $self->error(ERROR_FILE_READ, 'unable to read line', $!);
- }
-
- # Error on EOF (unless reading from error stream)
- if ($iBufferRead == 0 && (!defined($bIgnoreEOF) || !$bIgnoreEOF))
- {
- # Only error if reading from the input stream
- if (!defined($bError) || $bError)
- {
- $self->error(ERROR_FILE_READ, 'unexpected EOF');
- }
- # If reading from error stream then just return undef
- else
- {
- return;
- }
- }
- }
-
- # If data was read then check for a linefeed
- if ($iBufferRead > 0)
- {
- $self->{iBufferSize} += $iBufferRead;
-
- $iLineFeedPos = index($self->{strBuffer}, "\n");
- }
- else
- {
- $self->error();
- }
-
- # Calculate time remaining before timeout
- if ($iLineFeedPos == -1)
- {
- $fRemaining = $fTimeout - (gettimeofday() - $fTimeStart);
- }
- }
- while ($iLineFeedPos == -1 && $fRemaining > 0);
-
- # If not linefeed was found within the time return undef
- if ($iLineFeedPos == -1)
- {
- if (!defined($bError) || $bError)
- {
- confess &log(ERROR, "unable to read line after ${fTimeout} second(s)", ERROR_PROTOCOL_TIMEOUT);
- }
-
- return;
- }
- }
-
- # Return the line that was found and adjust the buffer position
- my $strLine = $iLineFeedPos - $self->{iBufferPos} == 0 ? '' :
- substr($self->{strBuffer}, $self->{iBufferPos}, $iLineFeedPos - $self->{iBufferPos});
- $self->{iBufferPos} = $iLineFeedPos + 1;
-
- return $strLine;
-}
-
-####################################################################################################################################
-# errorWrite
-#
-# Write error data to the stream.
-####################################################################################################################################
-sub errorWrite
-{
- my $self = shift;
- my $strBuffer = shift;
-
- if (defined($self->{pId}))
- {
- confess &log(ASSERT, 'errorWrite() not valid in master process');
- }
-
- $self->lineWrite($strBuffer, $self->{hError});
-}
-
-####################################################################################################################################
-# lineWrite
-#
-# Write line to the stream.
-####################################################################################################################################
-sub lineWrite
-{
- my $self = shift;
- my $strBuffer = shift;
- my $hOut = shift;
-
- # Check if the process has exited abnormally (doesn't seem like we should need this, but the next syswrite does a hard
- # abort if the remote process has already closed)
- $self->error();
-
- # Write the data
- my $iLineOut = syswrite(defined($hOut) ? $hOut : $self->{hOut}, (defined($strBuffer) ? $strBuffer : '') . "\n");
-
- if (!defined($iLineOut) || $iLineOut != (defined($strBuffer) ? length($strBuffer) : 0) + 1)
- {
- # Check if the process has exited abnormally
- $self->error(ERROR_PROTOCOL, "unable to write ${strBuffer}", $!);
- }
-}
-
-####################################################################################################################################
-# bufferRead
-#
-# Read data from a stream.
-####################################################################################################################################
-sub bufferRead
-{
- my $self = shift;
- my $tBufferRef = shift;
- my $iRequestSize = shift;
- my $iOffset = shift;
- my $bBlock = shift;
-
- # Set working variables
- my $iRemainingSize = $iRequestSize;
- $iOffset = defined($iOffset) ? $iOffset : 0;
-
- # If there is data left over in the buffer from lineRead then use it
- if (defined($self->{strBuffer}) && $self->{iBufferPos} < $self->{iBufferSize})
- {
- # There is enough data in the buffer to satisfy the entire request and have data left over
- if ($iRemainingSize < $self->{iBufferSize} - $self->{iBufferPos})
- {
- $$tBufferRef = substr($self->{strBuffer}, $self->{iBufferPos}, $iRequestSize);
- $self->{iBufferPos} += $iRequestSize;
- return $iRequestSize;
- }
- # Else the entire buffer will be used (and more may be needed if blocking)
- else
- {
- $$tBufferRef = substr($self->{strBuffer}, $self->{iBufferPos});
- my $iReadSize = $self->{iBufferSize} - $self->{iBufferPos};
- $iRemainingSize -= $iReadSize;
- undef($self->{strBuffer});
-
- return $iReadSize if $iRemainingSize == 0;
-
- $iOffset += $iReadSize;
- }
- }
-
- # If this is a blocking read then loop until all bytes have been read, else error. If not blocking read until the request size
- # has been met or EOF.
- my $fTimeStart = gettimeofday();
- my $fRemaining = 1; #$self->{iProtocolTimeout};
-
- do
- {
- # Check if the sysread call will block
- if ($self->{oInSelect}->can_read($fRemaining))
- {
- # Read a data into the buffer
- my $iReadSize = sysread($self->{hIn}, $$tBufferRef, $iRemainingSize, $iOffset);
-
- # Process errors from the sysread
- if (!defined($iReadSize))
- {
- $self->error(ERROR_FILE_READ, "unable to read ${iReadSize} bytes", $!);
- }
-
- # Check for EOF
- if ($iReadSize == 0)
- {
- if (defined($bBlock) && $bBlock)
- {
- confess &log(ERROR,
- 'EOF after ' . ($iRequestSize - $iRemainingSize) . " bytes but expected ${iRequestSize}",
- ERROR_FILE_READ);
- }
- else
- {
- return $iRequestSize - $iRemainingSize;
- }
- }
-
- # Update remaining size and return when it reaches 0
- $iRemainingSize -= $iReadSize;
-
- if ($iRemainingSize == 0)
- {
- return $iRequestSize;
- }
-
- # Update the offset to advance the next read further into the buffer
- $iOffset += $iReadSize;
- }
-
- # Calculate time remaining before timeout
- $fRemaining = $self->{iProtocolTimeout} - (gettimeofday() - $fTimeStart);
- }
- while ($fRemaining > 0);
-
- # Throw an error if timeout happened before required bytes were read
- confess &log(ERROR, "unable to read ${iRequestSize} bytes after $self->{iProtocolTimeout} seconds", ERROR_PROTOCOL_TIMEOUT);
-}
-
-####################################################################################################################################
-# bufferWrite
-#
-# Write data to a stream.
-####################################################################################################################################
-sub bufferWrite
-{
- my $self = shift;
- my $tBufferRef = shift;
- my $iWriteSize = shift;
-
- # If block size is not defined, get it from buffer length
- $iWriteSize = defined($iWriteSize) ? $iWriteSize : length($$tBufferRef);
-
- # Write the block
- my $iWriteOut = syswrite($self->{hOut}, $$tBufferRef, $iWriteSize);
-
- # Report any errors
- if (!defined($iWriteOut) || $iWriteOut != $iWriteSize)
- {
- $self->error(ERROR_FILE_WRITE, "unable to write ${iWriteSize} bytes", $!);
- }
-}
-
-####################################################################################################################################
-# error
-#
-# Format and confess error.
-####################################################################################################################################
-sub error
-{
- my $self = shift;
- my $iCode = shift;
- my $strMessage = shift;
- my $strSubMessage = shift;
-
- # Confess default error
- if (defined($iCode))
- {
- confess &log(ERROR, ($strMessage . (defined($strSubMessage) && $strSubMessage ne '' ? ": ${strSubMessage}" : '')), $iCode);
- }
-}
-
-1;
diff --git a/lib/pgBackRest/Protocol/Common/Io/Process.pm b/lib/pgBackRest/Protocol/Common/Io/Process.pm
deleted file mode 100644
index 159431df5..000000000
--- a/lib/pgBackRest/Protocol/Common/Io/Process.pm
+++ /dev/null
@@ -1,214 +0,0 @@
-####################################################################################################################################
-# PROTOCOL PROCESS IO MODULE
-####################################################################################################################################
-package pgBackRest::Protocol::Common::Io::Process;
-use parent 'pgBackRest::Protocol::Common::Io::Handle';
-
-use strict;
-use warnings FATAL => qw(all);
-use Carp qw(confess);
-use English '-no_match_vars';
-
-use Exporter qw(import);
- our @EXPORT = qw();
-use File::Basename qw(dirname);
-use IPC::Open3 qw(open3);
-use IO::Select;
-use POSIX qw(:sys_wait_h);
-use Symbol 'gensym';
-use Time::HiRes qw(gettimeofday);
-
-use pgBackRest::Common::Exception;
-use pgBackRest::Common::Log;
-use pgBackRest::Common::String;
-use pgBackRest::Common::Wait;
-use pgBackRest::Protocol::Common::Io::Handle;
-
-####################################################################################################################################
-# CONSTRUCTOR
-####################################################################################################################################
-sub new
-{
- my $class = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $hndIn, # Input stream
- $hndOut, # Output stream
- $hndErr, # Error stream
- $pId, # Process ID
- $strId, # Id for messages
- $iProtocolTimeout, # Protocol timeout
- $iBufferMax, # Maximum buffer size
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new', \@_,
- {name => 'hndIn', required => false, trace => true},
- {name => 'hndOut', required => false, trace => true},
- {name => 'hndErr', required => false, trace => true},
- {name => 'pId', required => false, trace => true},
- {name => 'strId', required => false, trace => true},
- {name => 'iProtocolTimeout', trace => true},
- {name => 'iBufferMax', trace => true}
- );
-
- my $self = $class->SUPER::new($hndIn, $hndOut, $hndErr, $iProtocolTimeout, $iBufferMax);
-
- $self->{pId} = $pId;
- $self->{strId} = $strId;
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'self', value => $self}
- );
-}
-
-####################################################################################################################################
-# new3
-#
-# Use open3 to run the command and get the io handles.
-####################################################################################################################################
-sub new3
-{
- my $class = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strId,
- $strCommand,
- $iProtocolTimeout, # Protocol timeout
- $iBufferMax # Maximum buffer Size
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new3', \@_,
- {name => 'strId', trace => true},
- {name => 'strCommand', trace => true},
- {name => 'iProtocolTimeout', trace => true},
- {name => 'iBufferMax', trace => true}
- );
-
- # Use open3 to run the command
- my ($pId, $hIn, $hOut, $hErr);
- $hErr = gensym;
-
- $pId = IPC::Open3::open3($hIn, $hOut, $hErr, $strCommand);
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'self', value => $class->new($hOut, $hIn, $hErr, $pId, $strId, $iProtocolTimeout, $iBufferMax)}
- );
-}
-
-####################################################################################################################################
-# Get process id
-####################################################################################################################################
-sub processId
-{
- my $self = shift;
-
- return $self->{pId};
-}
-
-####################################################################################################################################
-# error
-#
-# See if the remote process has terminated unexpectedly and attempt to retrieve error information from stderr. If that fails then
-# call parent error() method to report error.
-####################################################################################################################################
-sub error
-{
- my $self = shift;
- my $iCode = shift;
- my $strMessage = shift;
- my $strSubMessage = shift;
-
- # Record the start time and set initial sleep interval
- my $oWait = waitInit(defined($iCode) ? IO_ERROR_TIMEOUT : 0);
-
- if (defined($self->{pId}))
- {
- do
- {
- my $iResult = waitpid($self->{pId}, WNOHANG);
-
- # If there is no such process we'll assume it terminated previously
- if ($iResult == -1)
- {
- return true;
- }
-
- # If the process exited then this is unexpected
- if ($iResult > 0)
- {
- # Get the exit status so we can report it later
- my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
-
- # Initialize error
- my $strError = undef;
-
- # If the error stream is already closed then we can't fetch the real error
- if (!defined($self->{hErr}))
- {
- $strError = 'no error captured because stderr is already closed';
- }
- # Get whatever text we can from the error stream
- else
- {
- eval
- {
- while (my $strLine = $self->lineRead(0, false, false))
- {
- if (defined($strError))
- {
- $strError .= "\n";
- }
-
- $strError .= $strLine;
- }
-
- return true;
- }
- or do
- {
- if (!defined($strError))
- {
- my $strException = $EVAL_ERROR;
-
- $strError =
- 'no output from terminated process' .
- (defined($strException) && ${strException} ne '' ? ": ${strException}" : '');
- }
- };
- }
-
- $self->{pId} = undef;
- $self->{hIn} = undef;
- $self->{hOut} = undef;
- $self->{hErr} = undef;
-
- # Finally, confess the error
- confess &log(
- ERROR, 'remote process terminated on ' . $self->{strId} . ' host' .
- ($iExitStatus < ERROR_MINIMUM || $iExitStatus > ERROR_MAXIMUM ? " (exit status ${iExitStatus})" : '') .
- ': ' . (defined($strError) ? $strError : 'no error on stderr'),
- $iExitStatus >= ERROR_MINIMUM && $iExitStatus <= ERROR_MAXIMUM ? $iExitStatus : ERROR_HOST_CONNECT);
- }
- }
- while (waitMore($oWait));
- }
-
- # Confess default error
- $self->SUPER::error($iCode, $strMessage, $iCode);
-}
-
-1;
diff --git a/lib/pgBackRest/Protocol/Helper.pm b/lib/pgBackRest/Protocol/Helper.pm
index 6ae52971c..4475ea9ac 100644
--- a/lib/pgBackRest/Protocol/Helper.pm
+++ b/lib/pgBackRest/Protocol/Helper.pm
@@ -1,5 +1,5 @@
####################################################################################################################################
-# PROTOCOL HELPER MODULE
+# Create and manage protocol objects.
####################################################################################################################################
package pgBackRest::Protocol::Helper;
@@ -12,10 +12,72 @@ use Exporter qw(import);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Remote::Master;
use pgBackRest::Version;
+####################################################################################################################################
+# Operation constants
+####################################################################################################################################
+# Backup module
+use constant OP_BACKUP_FILE => 'backupFile';
+ push @EXPORT, qw(OP_BACKUP_FILE);
+
+# Archive Module
+use constant OP_ARCHIVE_GET_ARCHIVE_ID => 'archiveId';
+ push @EXPORT, qw(OP_ARCHIVE_GET_ARCHIVE_ID);
+use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';
+ push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);
+use constant OP_ARCHIVE_PUSH_CHECK => 'archivePushCheck';
+ push @EXPORT, qw(OP_ARCHIVE_PUSH_CHECK);
+
+# Archive Push Async Module
+use constant OP_ARCHIVE_PUSH_ASYNC => 'archivePushAsync';
+ push @EXPORT, qw(OP_ARCHIVE_PUSH_ASYNC);
+
+# Archive File Module
+use constant OP_ARCHIVE_PUSH_FILE => 'archivePushFile';
+ push @EXPORT, qw(OP_ARCHIVE_PUSH_FILE);
+
+# Check Module
+use constant OP_CHECK_BACKUP_INFO_CHECK => 'backupInfoCheck';
+ push @EXPORT, qw(OP_CHECK_BACKUP_INFO_CHECK);
+
+# Db Module
+use constant OP_DB_CONNECT => 'dbConnect';
+ push @EXPORT, qw(OP_DB_CONNECT);
+use constant OP_DB_EXECUTE_SQL => 'dbExecSql';
+ push @EXPORT, qw(OP_DB_EXECUTE_SQL);
+use constant OP_DB_INFO => 'dbInfo';
+ push @EXPORT, qw(OP_DB_INFO);
+
+# Storage Module
+use constant OP_STORAGE_OPEN_READ => 'storageOpenRead';
+ push @EXPORT, qw(OP_STORAGE_OPEN_READ);
+use constant OP_STORAGE_OPEN_WRITE => 'storageOpenWrite';
+ push @EXPORT, qw(OP_STORAGE_OPEN_WRITE);
+use constant OP_STORAGE_EXISTS => 'storageExists';
+ push @EXPORT, qw(OP_STORAGE_EXISTS);
+use constant OP_STORAGE_LIST => 'storageList';
+ push @EXPORT, qw(OP_STORAGE_LIST);
+use constant OP_STORAGE_MANIFEST => 'storageManifest';
+ push @EXPORT, qw(OP_STORAGE_MANIFEST);
+use constant OP_STORAGE_MOVE => 'storageMove';
+ push @EXPORT, qw(OP_STORAGE_MOVE);
+use constant OP_STORAGE_PATH_GET => 'storagePathGet';
+ push @EXPORT, qw(OP_STORAGE_PATH_GET);
+
+# Info module
+use constant OP_INFO_STANZA_LIST => 'infoStanzList';
+ push @EXPORT, qw(OP_INFO_STANZA_LIST);
+
+# Restore module
+use constant OP_RESTORE_FILE => 'restoreFile';
+ push @EXPORT, qw(OP_RESTORE_FILE);
+
+# Wait
+use constant OP_WAIT => 'wait';
+ push @EXPORT, qw(OP_WAIT);
+
####################################################################################################################################
# Module variables
####################################################################################################################################
@@ -29,9 +91,9 @@ my $hProtocol = {}; # Global remote hash that is created on first reques
sub isRepoLocal
{
# Not valid for remote
- if (commandTest(CMD_REMOTE))
+ if (commandTest(CMD_REMOTE) && !optionTest(OPTION_TYPE, BACKUP))
{
- confess &log(ASSERT, 'isRepoLocal() not valid on remote');
+ confess &log(ASSERT, 'isRepoLocal() not valid on ' . optionGet(OPTION_TYPE) . ' remote');
}
return optionTest(OPTION_BACKUP_HOST) ? false : true;
@@ -40,19 +102,37 @@ sub isRepoLocal
push @EXPORT, qw(isRepoLocal);
####################################################################################################################################
-# isDbLocal
-#
-# Is the database local?
+# isDbLocal - is the database local?
####################################################################################################################################
sub isDbLocal
{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $iRemoteIdx,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::isDbLocal', \@_,
+ {name => 'iRemoteIdx', optional => true, default => optionValid(OPTION_HOST_ID) ? optionGet(OPTION_HOST_ID) : 1,
+ trace => true},
+ );
+
# Not valid for remote
- if (commandTest(CMD_REMOTE))
+ if (commandTest(CMD_REMOTE) && !optionTest(OPTION_TYPE, DB))
{
- confess &log(ASSERT, 'isDbLocal() not valid on remote');
+ confess &log(ASSERT, 'isDbLocal() not valid on ' . optionGet(OPTION_TYPE) . ' remote');
}
- return optionTest(OPTION_DB_HOST) ? false : true;
+ my $bLocal = optionTest(optionIndex(OPTION_DB_HOST, $iRemoteIdx)) ? false : true;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bLocal', value => $bLocal, trace => true}
+ );
}
push @EXPORT, qw(isDbLocal);
@@ -80,7 +160,7 @@ sub protocolGet
(
__PACKAGE__ . '::protocolGet', \@_,
{name => 'strRemoteType'},
- {name => 'iRemoteIdx', default => 1},
+ {name => 'iRemoteIdx', default => optionValid(OPTION_HOST_ID) ? optionGet(OPTION_HOST_ID) : 1},
{name => 'bCache', optional => true, default => true},
{name => 'strBackRestBin', optional => true},
{name => 'iProcessIdx', optional => true},
@@ -91,19 +171,11 @@ sub protocolGet
my $oProtocol;
# If no remote requested or if the requested remote type is local then return a local protocol object
- my $strRemoteHost = $strRemoteType eq NONE ? undef : optionIndex("${strRemoteType}-host", $iRemoteIdx);
+ my $strRemoteHost = optionIndex("${strRemoteType}-host", $iRemoteIdx);
- if ($strRemoteType eq NONE || !optionTest($strRemoteHost))
+ if (!optionTest($strRemoteHost))
{
- logDebugMisc($strOperation, 'create local protocol');
-
- $oProtocol = new pgBackRest::Protocol::Common::Common
- (
- optionGet(OPTION_BUFFER_SIZE),
- commandTest(CMD_EXPIRE) ? OPTION_DEFAULT_COMPRESS_LEVEL : optionGet(OPTION_COMPRESS_LEVEL),
- commandTest(CMD_EXPIRE) ? OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK : optionGet(OPTION_COMPRESS_LEVEL_NETWORK),
- commandTest(CMD_EXPIRE) ? OPTION_PROTOCOL_TIMEOUT : optionGet(OPTION_PROTOCOL_TIMEOUT)
- );
+ confess &log(ASSERT, 'protocol cannot be created when remote host is not specified');
}
# Else create the remote protocol
else
@@ -116,6 +188,10 @@ sub protocolGet
{
$oProtocol = $$hProtocol{$strRemoteType}{$iRemoteIdx};
logDebugMisc($strOperation, 'found cached protocol');
+
+ # Issue a noop on protocol pulled from the cache to be sure it is still functioning. It's better to get an error at
+ # request time than somewhere randomly later.
+ $oProtocol->noOp();
}
# If protocol was not returned from cache then create it
@@ -157,7 +233,6 @@ sub protocolGet
$oProtocol = new pgBackRest::Protocol::Remote::Master
(
- $strRemoteType,
optionGet(OPTION_CMD_SSH),
commandWrite(
CMD_REMOTE, true,
@@ -210,6 +285,54 @@ sub protocolGet
push @EXPORT, qw(protocolGet);
+####################################################################################################################################
+# protocolList - list all active protocols
+####################################################################################################################################
+sub protocolList
+{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strRemoteType,
+ $iRemoteIdx,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::protocolList', \@_,
+ {name => 'strRemoteType', required => false, trace => true},
+ {name => 'iRemoteIdx', required => false, trace => true},
+ );
+
+ my @oyProtocol;
+
+ if (!defined($strRemoteType))
+ {
+ foreach my $strRemoteType (sort(keys(%{$hProtocol})))
+ {
+ push(@oyProtocol, protocolList($strRemoteType));
+ }
+ }
+ elsif (!defined($iRemoteIdx))
+ {
+ foreach my $iRemoteIdx (sort(keys(%{$hProtocol->{$strRemoteType}})))
+ {
+ push(@oyProtocol, protocolList($strRemoteType, $iRemoteIdx));
+ }
+ }
+ elsif (defined($hProtocol->{$strRemoteType}{$iRemoteIdx}))
+ {
+ push(@oyProtocol, {strRemoteType => $strRemoteType, iRemoteIdx => $iRemoteIdx});
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oyProtocol', value => \@oyProtocol, trace => true}
+ );
+}
+
####################################################################################################################################
# protocolDestroy
#
@@ -235,34 +358,15 @@ sub protocolDestroy
my $iExitStatus = 0;
- if (defined($strRemoteType))
+ foreach my $rhProtocol (protocolList($strRemoteType, $iRemoteIdx))
{
- $iRemoteIdx = defined($iRemoteIdx) ? $iRemoteIdx : 1;
+ logDebugMisc(
+ $strOperation, 'found cached protocol',
+ {name => 'strRemoteType', value => $rhProtocol->{strRemoteType}},
+ {name => 'iRemoteIdx', value => $rhProtocol->{iRemoteIdx}});
- if (defined($$hProtocol{$strRemoteType}{$iRemoteIdx}))
- {
- $iExitStatus = ($$hProtocol{$strRemoteType}{$iRemoteIdx})->close($bComplete);
- delete($$hProtocol{$strRemoteType}{$iRemoteIdx});
- }
- }
- else
- {
- foreach my $strRemoteType (sort(keys(%{$hProtocol})))
- {
- foreach my $iRemoteIdx (sort(keys(%{$$hProtocol{$strRemoteType}})))
- {
- if (defined($$hProtocol{$strRemoteType}{$iRemoteIdx}))
- {
- logDebugMisc(
- $strOperation, 'found cached protocol',
- {name => 'strRemoteType', value => $strRemoteType},
- {name => 'iRemoteIdx', value => $iRemoteIdx});
-
- $iExitStatus = ($$hProtocol{$strRemoteType}{$iRemoteIdx})->close($bComplete);
- delete($$hProtocol{$strRemoteType}{$iRemoteIdx});
- }
- }
- }
+ $iExitStatus = $hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}}->close($bComplete);
+ delete($hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}});
}
# Return from function and log return values if any
@@ -275,4 +379,34 @@ sub protocolDestroy
push @EXPORT, qw(protocolDestroy);
+####################################################################################################################################
+# protocolKeepAlive - call keepAlive() on all protocols
+####################################################################################################################################
+sub protocolKeepAlive
+{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strRemoteType,
+ $iRemoteIdx,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::protocolDestroy', \@_,
+ {name => 'strRemoteType', required => false, trace => true},
+ {name => 'iRemoteIdx', required => false, trace => true},
+ );
+
+ foreach my $rhProtocol (protocolList($strRemoteType, $iRemoteIdx))
+ {
+ $hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}}->keepAlive();
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+push @EXPORT, qw(protocolKeepAlive);
+
1;
diff --git a/lib/pgBackRest/Protocol/Local/Master.pm b/lib/pgBackRest/Protocol/Local/Master.pm
index 6698a8aef..2a300b930 100644
--- a/lib/pgBackRest/Protocol/Local/Master.pm
+++ b/lib/pgBackRest/Protocol/Local/Master.pm
@@ -12,7 +12,6 @@ use pgBackRest::Backup::File;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Command::Master;
-use pgBackRest::Protocol::Common::Common;
####################################################################################################################################
# CONSTRUCTOR
@@ -36,17 +35,12 @@ sub new
);
# Init object and store variables
- my $strLocal = 'local-' . $iProcessIdx;
-
my $self = $class->SUPER::new(
- NONE, 'local', $strLocal, $strCommand, optionGet(OPTION_BUFFER_SIZE), optionGet(OPTION_COMPRESS_LEVEL),
+ 'local', "'local-${iProcessIdx}'", $strCommand, optionGet(OPTION_BUFFER_SIZE), optionGet(OPTION_COMPRESS_LEVEL),
optionGet(OPTION_COMPRESS_LEVEL_NETWORK), optionGet(OPTION_PROTOCOL_TIMEOUT));
bless $self, $class;
- # Store the host
- $self->{strLocal} = $strLocal;
-
# Return from function and log return values if any
return logDebugReturn
(
diff --git a/lib/pgBackRest/Protocol/Local/Minion.pm b/lib/pgBackRest/Protocol/Local/Minion.pm
index 471129ac6..b52f14527 100644
--- a/lib/pgBackRest/Protocol/Local/Minion.pm
+++ b/lib/pgBackRest/Protocol/Local/Minion.pm
@@ -12,9 +12,9 @@ use pgBackRest::Archive::ArchivePushFile;
use pgBackRest::Backup::File;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
-use pgBackRest::File;
+use pgBackRest::Storage::Local;
+use pgBackRest::Protocol::Base::Master;
use pgBackRest::Protocol::Command::Minion;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
use pgBackRest::RestoreFile;
@@ -26,21 +26,10 @@ sub new
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $strCommand,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->new', \@_,
- {name => 'strCommand'},
- );
+ my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
# Init object and store variables
- my $self = $class->SUPER::new(
- CMD_LOCAL, $strCommand, optionGet(OPTION_BUFFER_SIZE), optionGet(OPTION_COMPRESS_LEVEL),
- optionGet(OPTION_COMPRESS_LEVEL_NETWORK), optionGet(OPTION_PROTOCOL_TIMEOUT));
+ my $self = $class->SUPER::new(CMD_LOCAL, optionGet(OPTION_BUFFER_SIZE), optionGet(OPTION_PROTOCOL_TIMEOUT));
bless $self, $class;
# Return from function and log return values if any
@@ -61,25 +50,12 @@ sub init
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');
- # Create the file object
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(
- optionGet(OPTION_TYPE), optionGet(OPTION_HOST_ID),
- {iProcessIdx => optionGet(OPTION_PROCESS), strCommand => optionGet(OPTION_COMMAND)})
- );
-
# Create anonymous subs for each command
my $hCommandMap =
{
- &OP_ARCHIVE_PUSH_FILE => sub {archivePushFile($oFile, @{shift()})},
- &OP_BACKUP_FILE => sub {backupFile($oFile, @{shift()})},
- &OP_RESTORE_FILE => sub {restoreFile($oFile, @{shift()})},
-
- # To be run after each command to keep the remote alive
- &OP_POST => sub {$oFile->{oProtocol}->keepAlive()},
+ &OP_ARCHIVE_PUSH_FILE => sub {archivePushFile(@{shift()})},
+ &OP_BACKUP_FILE => sub {backupFile(@{shift()})},
+ &OP_RESTORE_FILE => sub {restoreFile(@{shift()})},
};
# Return from function and log return values if any
diff --git a/lib/pgBackRest/Protocol/Local/Process.pm b/lib/pgBackRest/Protocol/Local/Process.pm
index 802143f3f..d8a51cb86 100644
--- a/lib/pgBackRest/Protocol/Local/Process.pm
+++ b/lib/pgBackRest/Protocol/Local/Process.pm
@@ -197,7 +197,7 @@ sub hostConnect
iProcessId => $iProcessId,
iHostProcessIdx => $iHostProcessIdx,
oLocal => $oLocal,
- hndIn => fileno($oLocal->{io}->{hIn}),
+ hndIn => fileno($oLocal->io()->handleRead()),
};
push(@{$self->{hyLocal}}, $hLocal);
@@ -335,7 +335,7 @@ sub process
confess $oException if (!isException($oException));
# If the process is has terminated throw the exception
- if (!defined($hLocal->{oLocal}->{io}->processId()))
+ if (!defined($hLocal->{oLocal}->io()->processId()))
{
confess logException($oException);
}
diff --git a/lib/pgBackRest/Protocol/Remote/Master.pm b/lib/pgBackRest/Protocol/Remote/Master.pm
index 69f520747..8c3ea2032 100644
--- a/lib/pgBackRest/Protocol/Remote/Master.pm
+++ b/lib/pgBackRest/Protocol/Remote/Master.pm
@@ -25,7 +25,6 @@ sub new
my
(
$strOperation,
- $strRemoteType, # Type of remote (DB or BACKUP)
$strCommandSSH, # SSH client
$strCommand, # Command to execute on local/remote
$iBufferMax, # Maximum buffer size
@@ -38,7 +37,6 @@ sub new
logDebugParam
(
__PACKAGE__ . '->new', \@_,
- {name => 'strRemoteType'},
{name => 'strCommandSSH'},
{name => 'strCommand'},
{name => 'iBufferMax'},
@@ -55,7 +53,8 @@ sub new
# Init object and store variables
my $self = $class->SUPER::new(
- $strRemoteType, 'remote', $strHost, $strCommand, $iBufferMax, $iCompressLevel, $iCompressLevelNetwork, $iProtocolTimeout);
+ 'remote', "'$strHost remote'", $strCommand, $iBufferMax, $iCompressLevel, $iCompressLevelNetwork,
+ $iProtocolTimeout);
bless $self, $class;
# Store the host
diff --git a/lib/pgBackRest/Protocol/Remote/Minion.pm b/lib/pgBackRest/Protocol/Remote/Minion.pm
index d01dce763..de9b1b8f5 100644
--- a/lib/pgBackRest/Protocol/Remote/Minion.pm
+++ b/lib/pgBackRest/Protocol/Remote/Minion.pm
@@ -12,15 +12,17 @@ use File::Basename qw(dirname);
use pgBackRest::Backup::File;
use pgBackRest::Common::Log;
+use pgBackRest::Common::Io::Buffered;
+use pgBackRest::Common::Wait;
use pgBackRest::Archive::ArchiveGet;
use pgBackRest::Archive::ArchivePushFile;
use pgBackRest::Check::Check;
use pgBackRest::Config::Config;
use pgBackRest::Db;
-use pgBackRest::File;
use pgBackRest::Info;
use pgBackRest::Protocol::Command::Minion;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
####################################################################################################################################
# CONSTRUCTOR
@@ -33,25 +35,18 @@ sub new
my
(
$strOperation,
- $strCommand, # Command the master process is running
$iBufferMax, # Maximum buffer size
- $iCompressLevel, # Set compression level
- $iCompressLevelNetwork, # Set compression level for network only compression,
$iProtocolTimeout # Protocol timeout
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
- {name => 'strCommand'},
{name => 'iBufferMax'},
- {name => 'iCompressLevel'},
- {name => 'iCompressNetworkLevel'},
{name => 'iProtocolTimeout'}
);
# Init object and store variables
- my $self = $class->SUPER::new(CMD_REMOTE, $strCommand, $iBufferMax, $iCompressLevel,
- $iCompressLevelNetwork, $iProtocolTimeout);
+ my $self = $class->SUPER::new(CMD_REMOTE, $iBufferMax, $iProtocolTimeout);
bless $self, $class;
# Return from function and log return values if any
@@ -73,30 +68,25 @@ sub init
my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');
# Create objects
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA, false),
- optionGet(OPTION_REPO_PATH, false),
- $self
- );
+ my $oStorage = optionTest(OPTION_TYPE, DB) ? storageDb() : storageRepo();
- my $oArchiveGet = new pgBackRest::Archive::ArchiveGet();
- my $oCheck = new pgBackRest::Check::Check();
- my $oInfo = new pgBackRest::Info();
- my $oDb = new pgBackRest::Db();
+ my $oArchiveGet = optionTest(OPTION_TYPE, BACKUP) ? new pgBackRest::Archive::ArchiveGet() : undef;
+ my $oCheck = optionTest(OPTION_TYPE, BACKUP) ? new pgBackRest::Check::Check() : undef;
+ my $oInfo = optionTest(OPTION_TYPE, BACKUP) ? new pgBackRest::Info() : undef;
+ my $oDb = optionTest(OPTION_TYPE, DB) ? new pgBackRest::Db() : undef;
# Create anonymous subs for each command
my $hCommandMap =
{
# ArchiveGet commands
- &OP_ARCHIVE_GET_ARCHIVE_ID => sub {$oArchiveGet->getArchiveId($oFile)},
- &OP_ARCHIVE_GET_CHECK => sub {$oArchiveGet->getCheck($oFile, @{shift()})},
+ &OP_ARCHIVE_GET_ARCHIVE_ID => sub {$oArchiveGet->getArchiveId()},
+ &OP_ARCHIVE_GET_CHECK => sub {$oArchiveGet->getCheck(@{shift()})},
# ArchivePush commands
- &OP_ARCHIVE_PUSH_CHECK => sub {archivePushCheck($oFile, @{shift()})},
+ &OP_ARCHIVE_PUSH_CHECK => sub {archivePushCheck(@{shift()})},
# Check commands
- &OP_CHECK_BACKUP_INFO_CHECK => sub {$oCheck->backupInfoCheck($oFile, @{shift()})},
+ &OP_CHECK_BACKUP_INFO_CHECK => sub {$oCheck->backupInfoCheck(@{shift()})},
# Db commands
&OP_DB_CONNECT => sub {$oDb->connect()},
@@ -104,17 +94,38 @@ sub init
&OP_DB_INFO => sub {$oDb->info(@{shift()})},
# File commands
- &OP_FILE_COPY => sub {my $rParam = shift; $oFile->copy(PATH_ABSOLUTE, shift(@{$rParam}), PATH_ABSOLUTE, @{$rParam})},
- &OP_FILE_COPY_IN => sub {my $rParam = shift; $oFile->copy(PIPE_STDIN, shift(@{$rParam}), PATH_ABSOLUTE, @{$rParam})},
- &OP_FILE_COPY_OUT => sub {my $rParam = shift; $oFile->copy(PATH_ABSOLUTE, shift(@{$rParam}), PIPE_STDOUT, @{$rParam})},
- &OP_FILE_EXISTS => sub {$oFile->exists(PATH_ABSOLUTE, @{shift()})},
- &OP_FILE_LIST => sub {$oFile->list(PATH_ABSOLUTE, @{shift()})},
- &OP_FILE_MANIFEST => sub {$oFile->manifest(PATH_ABSOLUTE, @{shift()})},
- &OP_FILE_PATH_CREATE => sub {$oFile->pathCreate(PATH_ABSOLUTE, @{shift()})},
- &OP_FILE_WAIT => sub {$oFile->wait(PATH_ABSOLUTE, @{shift()})},
+ &OP_STORAGE_OPEN_READ => sub
+ {
+ my $oSourceFileIo = $oStorage->openRead(@{shift()});
+
+ # If the source file exists
+ if (defined($oSourceFileIo))
+ {
+ $self->outputWrite(true);
+
+ $oStorage->copy($oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo));
+
+ return true;
+ }
+
+ return false;
+ },
+ &OP_STORAGE_OPEN_WRITE => sub
+ {
+ my $oDestinationFileIo = $oStorage->openWrite(@{shift()});
+ $oStorage->copy(new pgBackRest::Protocol::Storage::File($self, $oDestinationFileIo), $oDestinationFileIo);
+ },
+ &OP_STORAGE_EXISTS => sub {$oStorage->exists(@{shift()})},
+ &OP_STORAGE_LIST => sub {$oStorage->list(@{shift()})},
+ &OP_STORAGE_MANIFEST => sub {$oStorage->manifest(@{shift()})},
+ &OP_STORAGE_MOVE => sub {$oStorage->move(@{shift()})},
+ &OP_STORAGE_PATH_GET => sub {$oStorage->pathGet(@{shift()})},
# Info commands
- &OP_INFO_STANZA_LIST => sub {$oInfo->stanzaList($oFile, @{shift()})},
+ &OP_INFO_STANZA_LIST => sub {$oInfo->stanzaList(@{shift()})},
+
+ # Wait command
+ &OP_WAIT => sub {waitRemainder(@{shift()})},
};
# Return from function and log return values if any
diff --git a/lib/pgBackRest/Protocol/Storage/File.pm b/lib/pgBackRest/Protocol/Storage/File.pm
new file mode 100644
index 000000000..e93ccb986
--- /dev/null
+++ b/lib/pgBackRest/Protocol/Storage/File.pm
@@ -0,0 +1,151 @@
+####################################################################################################################################
+# Protocol File
+####################################################################################################################################
+package pgBackRest::Protocol::Storage::File;
+use parent 'pgBackRest::Common::Io::Base';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $oProtocol, # Master or minion protocol
+ $oFileIo, # File whose results will be returned via protocol
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'oProtocol', trace => true},
+ {name => 'oFileIo', required => false, trace => true},
+ );
+
+ # Create class
+ my $self = $class->SUPER::new($oProtocol->io()->id() . ' file');
+ bless $self, $class;
+
+ # Set variables
+ $self->{oProtocol} = $oProtocol;
+ $self->{oFileIo} = $oFileIo;
+
+ # Set read/write
+ $self->{bWrite} = false;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# read - read block from protocol
+####################################################################################################################################
+sub read
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+
+ my $lBlockSize;
+
+ # Read the block header and make sure it's valid
+ my $strBlockHeader = $self->{oProtocol}->io()->readLine();
+
+ if ($strBlockHeader !~ /^BRBLOCK[0-9]+$/)
+ {
+ confess &log(ERROR, "invalid block header '${strBlockHeader}'", ERROR_FILE_READ);
+ }
+
+ # Get block size from the header
+ $lBlockSize = substr($strBlockHeader, 7);
+
+ # Read block if size > 0
+ if ($lBlockSize > 0)
+ {
+ $self->{oProtocol}->io()->read($rtBuffer, $lBlockSize, true);
+ }
+
+ # Return the block size
+ return $lBlockSize;
+}
+
+####################################################################################################################################
+# write - write block to protocol
+####################################################################################################################################
+sub write
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+
+ # Set write
+ $self->{bWrite} = true;
+
+ # Get the buffer size
+ my $lBlockSize = defined($rtBuffer) ? length($$rtBuffer) : 0;
+
+ # Write if size > 0 (0 ends the copy stream so it should only be done in close())
+ if ($lBlockSize > 0)
+ {
+ # Write block header to the protocol stream
+ $self->{oProtocol}->io()->writeLine("BRBLOCK${lBlockSize}");
+
+ # Write block if size
+ $self->{oProtocol}->io()->write($rtBuffer);
+ }
+
+ return length($$rtBuffer);
+}
+
+####################################################################################################################################
+# close - set the result hash
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ # Close if protocol is defined (to prevent this from running more than once)
+ if (defined($self->{oProtocol}))
+ {
+ # If writing output terminator
+ if ($self->{bWrite})
+ {
+ $self->{oProtocol}->io()->writeLine("BRBLOCK0");
+ }
+
+ # On master read the results
+ if ($self->{oProtocol}->master())
+ {
+ ($self->{rhResult}) = $self->{oProtocol}->outputRead();
+
+ # Minion will send one more output message after file is closed which can be ignored
+ $self->{oProtocol}->outputRead();
+ }
+ # On minion write the results
+ else
+ {
+ $self->{oProtocol}->outputWrite($self->{oFileIo}->{rhResult});
+ }
+
+ # Delete protocol to prevent close from running again
+ delete($self->{oProtocol});
+ }
+}
+
+1;
diff --git a/lib/pgBackRest/Protocol/Storage/Helper.pm b/lib/pgBackRest/Protocol/Storage/Helper.pm
new file mode 100644
index 000000000..c2b299b80
--- /dev/null
+++ b/lib/pgBackRest/Protocol/Storage/Helper.pm
@@ -0,0 +1,220 @@
+####################################################################################################################################
+# Db & Repository Storage Helper
+####################################################################################################################################
+package pgBackRest::Protocol::Storage::Helper;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use File::Basename qw(basename);
+
+use pgBackRest::Common::Log;
+use pgBackRest::Config::Config;
+use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Remote;
+use pgBackRest::Storage::Helper;
+use pgBackRest::Storage::Local;
+
+####################################################################################################################################
+# Storage constants
+####################################################################################################################################
+use constant STORAGE_DB => '';
+ push @EXPORT, qw(STORAGE_DB);
+
+use constant STORAGE_REPO => '';
+ push @EXPORT, qw(STORAGE_REPO);
+use constant STORAGE_REPO_ARCHIVE => '';
+ push @EXPORT, qw(STORAGE_REPO_ARCHIVE);
+use constant STORAGE_REPO_BACKUP => '';
+ push @EXPORT, qw(STORAGE_REPO_BACKUP);
+
+####################################################################################################################################
+# Cache storage so it can be retrieved quickly
+####################################################################################################################################
+my $hStorage;
+
+####################################################################################################################################
+# storageDb - get db storage
+####################################################################################################################################
+sub storageDb
+{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $iRemoteIdx,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::storageDb', \@_,
+ {name => 'iRemoteIdx', optional => true, default => optionValid(OPTION_HOST_ID) ? optionGet(OPTION_HOST_ID) : 1,
+ trace => true},
+ );
+
+ # Create storage if not defined
+ if (!defined($hStorage->{&STORAGE_DB}{$iRemoteIdx}))
+ {
+ if (isDbLocal({iRemoteIdx => $iRemoteIdx}))
+ {
+ $hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Local(
+ optionGet(optionIndex(OPTION_DB_PATH, $iRemoteIdx)), new pgBackRest::Storage::Posix::Driver(),
+ {strTempExtension => STORAGE_TEMP_EXT, lBufferMax => optionGet(OPTION_BUFFER_SIZE)});
+ }
+ else
+ {
+ $hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Protocol::Storage::Remote(protocolGet(DB, $iRemoteIdx));
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oStorageDb', value => $hStorage->{&STORAGE_DB}{$iRemoteIdx}, trace => true},
+ );
+}
+
+push @EXPORT, qw(storageDb);
+
+####################################################################################################################################
+# storageRepoRule - rules for paths in the repository
+####################################################################################################################################
+sub storageRepoRule
+{
+ my $strRule = shift;
+ my $strFile = shift;
+ my $strStanza = shift;
+
+ # Result path and file
+ my $strResultFile;
+
+ # Return archive path
+ if ($strRule eq STORAGE_REPO_ARCHIVE)
+ {
+ $strResultFile = "archive/${strStanza}";
+
+ # If file is not defined nothing further to do
+ if (defined($strFile))
+ {
+ my ($strArchiveId, $strWalFile) = split('/', $strFile);
+
+ # If a WAL file (including .backup)
+ if (defined($strWalFile) && $strWalFile =~ /^[0-F]{24}/)
+ {
+ $strResultFile .= "/${strArchiveId}/" . substr($strWalFile, 0, 16) . "/${strWalFile}";
+ }
+ # Else other files
+ else
+ {
+ $strResultFile .= "/${strFile}";
+ }
+ }
+ }
+ # Return backup path
+ elsif ($strRule eq STORAGE_REPO_BACKUP)
+ {
+ $strResultFile = "backup/${strStanza}" . (defined($strFile) ? "/${strFile}" : '');
+ }
+ # Else error
+ else
+ {
+ confess &log(ASSERT, "invalid " . STORAGE_REPO . " storage rule ${strRule}");
+ }
+
+ return $strResultFile;
+}
+
+####################################################################################################################################
+# storageRepo - get repository storage
+####################################################################################################################################
+sub storageRepo
+{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strStanza,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::storageRepo', \@_,
+ {name => 'strStanza', optional => true, trace => true},
+ );
+
+ if (!defined($strStanza))
+ {
+ if (optionValid(OPTION_STANZA) && optionTest(OPTION_STANZA))
+ {
+ $strStanza = optionGet(OPTION_STANZA);
+ }
+ else
+ {
+ $strStanza = STORAGE_REPO;
+ }
+ }
+
+ # Create storage if not defined
+ if (!defined($hStorage->{&STORAGE_REPO}{$strStanza}))
+ {
+ if (isRepoLocal())
+ {
+ # Path rules
+ my $hRule;
+
+ if ($strStanza ne STORAGE_REPO)
+ {
+ $hRule =
+ {
+ &STORAGE_REPO_ARCHIVE =>
+ {
+ fnRule => \&storageRepoRule,
+ xData => $strStanza,
+ },
+ &STORAGE_REPO_BACKUP =>
+ {
+ fnRule => \&storageRepoRule,
+ xData => $strStanza,
+ },
+ }
+ };
+
+ # Create the driver
+ my $oDriver;
+
+ if (optionTest(OPTION_REPO_TYPE, REPO_TYPE_CIFS))
+ {
+ require pgBackRest::Storage::Cifs::Driver;
+
+ $oDriver = new pgBackRest::Storage::Cifs::Driver();
+ }
+ else
+ {
+ $oDriver = new pgBackRest::Storage::Posix::Driver();
+ }
+
+ # Create local storage
+ $hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Storage::Local(
+ optionGet(OPTION_REPO_PATH), $oDriver,
+ {strTempExtension => STORAGE_TEMP_EXT, hRule => $hRule, lBufferMax => optionGet(OPTION_BUFFER_SIZE)});
+ }
+ else
+ {
+ # Create remote storage
+ $hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Protocol::Storage::Remote(protocolGet(BACKUP));
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}{$strStanza}, trace => true},
+ );
+}
+
+push @EXPORT, qw(storageRepo);
+
+1;
diff --git a/lib/pgBackRest/Protocol/Storage/Remote.pm b/lib/pgBackRest/Protocol/Storage/Remote.pm
new file mode 100644
index 000000000..dc8a0cf89
--- /dev/null
+++ b/lib/pgBackRest/Protocol/Storage/Remote.pm
@@ -0,0 +1,315 @@
+####################################################################################################################################
+# Remote Storage
+####################################################################################################################################
+package pgBackRest::Protocol::Storage::Remote;
+use parent 'pgBackRest::Storage::Base';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Config::Config;
+use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::File;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Filter::Gzip;
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $oProtocol,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'oProtocol'},
+ );
+
+ # Init parent
+ my $self = $class->SUPER::new({lBufferMax => $oProtocol->io()->bufferMax()});
+ bless $self, $class;
+
+ # Set variables
+ $self->{oProtocol} = $oProtocol;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# exists
+####################################################################################################################################
+sub exists
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->exists', \@_,
+ {name => 'strPathExp'},
+ );
+
+ my $bExists = $self->{oProtocol}->cmdExecute(OP_STORAGE_EXISTS, [$strPathExp]);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bExists', value => $bExists}
+ );
+}
+
+####################################################################################################################################
+# list
+####################################################################################################################################
+sub list
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ $rhParam,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->list', \@_,
+ {name => 'strPathExp'},
+ {name => 'rhParam', required => false},
+ );
+
+ my @stryFileList = $self->{oProtocol}->cmdExecute(OP_STORAGE_LIST, [$strPathExp, $rhParam]);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'stryFileList', value => \@stryFileList}
+ );
+}
+
+####################################################################################################################################
+# manifest
+####################################################################################################################################
+sub manifest
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->manifest', \@_,
+ {name => 'strPathExp'},
+ );
+
+ my $hManifest = $self->{oProtocol}->cmdExecute(OP_STORAGE_MANIFEST, [$strPathExp]);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'hManifest', value => $hManifest, trace => true}
+ );
+}
+
+####################################################################################################################################
+# openRead
+####################################################################################################################################
+sub openRead
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFileExp,
+ $rhParam,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->openRead', \@_,
+ {name => 'strFileExp'},
+ {name => 'rhParam', required => false},
+ );
+
+ my $bProtocolCompress = defined($rhParam->{bProtocolCompress}) && $rhParam->{bProtocolCompress};
+
+ # Compress on the remote side
+ if ($bProtocolCompress)
+ {
+ push(
+ @{$rhParam->{rhyFilter}},
+ {strClass => STORAGE_FILTER_GZIP,
+ rxyParam => [{iLevel => optionGet(OPTION_COMPRESS_LEVEL_NETWORK), bWantGzip => false}]});
+ delete($rhParam->{bProtocolCompress});
+ }
+
+ my $oSourceFileIo =
+ $self->{oProtocol}->cmdExecute(OP_STORAGE_OPEN_READ, [$strFileExp, $rhParam]) ?
+ new pgBackRest::Protocol::Storage::File($self->{oProtocol}) : undef;
+
+ # Decompress on the local side
+ if ($bProtocolCompress)
+ {
+ $oSourceFileIo = new pgBackRest::Storage::Filter::Gzip(
+ $oSourceFileIo, {strCompressType => STORAGE_DECOMPRESS, bWantGzip => false});
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oFileIo', value => $oSourceFileIo, trace => true},
+ );
+}
+
+####################################################################################################################################
+# openWrite
+####################################################################################################################################
+sub openWrite
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFileExp,
+ $rhParam,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->openWrite', \@_,
+ {name => 'strFileExp'},
+ {name => 'rhParam', required => false},
+ );
+
+ # Is protocol compression enabled?
+ my $bProtocolCompress = defined($rhParam->{bProtocolCompress}) && $rhParam->{bProtocolCompress};
+
+ # Decompress on the remote side
+ if ($bProtocolCompress)
+ {
+ push(
+ @{$rhParam->{rhyFilter}},
+ {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS, bWantGzip => false}]});
+ delete($rhParam->{bProtocolCompress});
+ }
+
+ # Open the remote file
+ $self->{oProtocol}->cmdWrite(OP_STORAGE_OPEN_WRITE, [$strFileExp, $rhParam]);
+ my $oDestinationFileIo = new pgBackRest::Protocol::Storage::File($self->{oProtocol});
+
+ # Compress on local side
+ if ($bProtocolCompress)
+ {
+ $oDestinationFileIo = new pgBackRest::Storage::Filter::Gzip(
+ $oDestinationFileIo, {iLevel => optionGet(OPTION_COMPRESS_LEVEL_NETWORK), bWantGzip => false});
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oFileIo', value => $oDestinationFileIo, trace => true},
+ );
+}
+
+####################################################################################################################################
+# pathGet
+####################################################################################################################################
+sub pathGet
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ $rhParam,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathGet', \@_,
+ {name => 'strPathExp', required => false},
+ {name => 'rhParam', required => false},
+ );
+
+ my $strPath = $self->{oProtocol}->cmdExecute(OP_STORAGE_PATH_GET, [$strPathExp, $rhParam]);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'strPath', value => $strPath}
+ );
+}
+
+####################################################################################################################################
+# move
+####################################################################################################################################
+sub move
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strSourcePathExp,
+ $strDestinationPathExp,
+ $rhParam,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->move', \@_,
+ {name => 'strSourcePathExp'},
+ {name => 'strDestinationPathExp'},
+ {name => 'rhParam', required => false},
+ );
+
+ $self->{oProtocol}->cmdExecute(OP_STORAGE_MOVE, [$strSourcePathExp, $strDestinationPathExp, $rhParam], false);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation
+ );
+}
+
+####################################################################################################################################
+# getters
+####################################################################################################################################
+sub protocol {shift->{oProtocol}};
+
+1;
diff --git a/lib/pgBackRest/Restore.pm b/lib/pgBackRest/Restore.pm
index 3c22532dc..e99c9650e 100644
--- a/lib/pgBackRest/Restore.pm
+++ b/lib/pgBackRest/Restore.pm
@@ -17,13 +17,12 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::RestoreFile;
-use pgBackRest::Protocol::Common::Common;
-use pgBackRest::Protocol::Local::Process;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Local::Process;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
@@ -41,15 +40,7 @@ sub new
my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
# Initialize protocol
- $self->{oProtocol} = protocolGet(BACKUP);
-
- # Initialize default file object
- $self->{oFile} = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- $self->{oProtocol}
- );
+ $self->{oProtocol} = !isRepoLocal() ? protocolGet(BACKUP) : undef;
# Initialize variables
$self->{strDbClusterPath} = optionGet(OPTION_DB_PATH);
@@ -63,22 +54,6 @@ sub new
);
}
-####################################################################################################################################
-# DESTROY
-####################################################################################################################################
-sub DESTROY
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my ($strOperation) = logDebugParam(__PACKAGE__ . '->DESTROY');
-
- undef($self->{oFile});
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
-}
-
####################################################################################################################################
# manifestOwnershipCheck
#
@@ -226,18 +201,19 @@ sub manifestLoad
my ($strOperation) = logDebugParam (__PACKAGE__ . '->manifestLoad');
# Error if the backup set does not exist
- if (!$self->{oFile}->exists(PATH_BACKUP_CLUSTER, $self->{strBackupSet}))
+ if (!storageRepo()->exists(STORAGE_REPO_BACKUP . "/$self->{strBackupSet}/" . FILE_MANIFEST))
{
- confess &log(ERROR, 'backup ' . $self->{strBackupSet} . ' does not exist');
+ confess &log(ERROR, "backup '$self->{strBackupSet}' does not exist");
}
# Copy the backup manifest to the db cluster path
- $self->{oFile}->copy(PATH_BACKUP_CLUSTER, "$self->{strBackupSet}/" . FILE_MANIFEST,
- PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
+ storageDb()->copy(
+ storageRepo()->openRead(STORAGE_REPO_BACKUP . "/$self->{strBackupSet}/" . FILE_MANIFEST, {bProtocolCompress => true}),
+ $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
# Load the manifest into a hash
- my $oManifest = new pgBackRest::Manifest($self->{oFile}->pathGet(PATH_DB_ABSOLUTE,
- $self->{strDbClusterPath} . '/' . FILE_MANIFEST));
+ my $oManifest = new pgBackRest::Manifest(
+ storageDb()->pathGet($self->{strDbClusterPath} . '/' . FILE_MANIFEST), undef, {oStorage => storageDb()});
# If backup is latest then set it equal to backup label, else verify that requested backup and label match
my $strBackupLabel = $oManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL);
@@ -486,6 +462,9 @@ sub clean
{name => 'oManifest'}
);
+ # Db storage
+ my $oStorageDb = storageDb();
+
# Track if files/links/paths where removed
my %oRemoveHash =
(
@@ -515,7 +494,7 @@ sub clean
$strBasePath .= "/${strTargetPath}";
}
- ${$self->{oTargetPath}}{$strTarget} = pathAbsolute($strBasePath, $strCheckPath);
+ ${$self->{oTargetPath}}{$strTarget} = $oStorageDb->pathAbsolute($strBasePath, $strCheckPath);
$strCheckPath = ${$self->{oTargetPath}}{$strTarget};
@@ -528,7 +507,7 @@ sub clean
&log(DETAIL, "check ${strCheckPath} exists");
# Check if the path exists
- if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strCheckPath))
+ if (!$oStorageDb->pathExists($strCheckPath))
{
confess &log(ERROR, "cannot restore to missing path ${strCheckPath}", ERROR_PATH_MISSING);
}
@@ -541,7 +520,7 @@ sub clean
$oManifest->get(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE);
# Check if the file exists
- if ($self->{oFile}->exists(PATH_DB_ABSOLUTE, $strCheckFile))
+ if ($oStorageDb->exists($strCheckFile))
{
# If the file exists and this is not a delta then error
if (!$bDelta)
@@ -564,14 +543,14 @@ sub clean
${$self->{oTargetPath}}{$strTarget} = "${$self->{oTargetPath}}{$strTarget}/" . $oManifest->tablespacePathGet();
# If this path does not exist then skip the rest of the checking - the path will be created later
- if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, ${$self->{oTargetPath}}{$strTarget}))
+ if (!$oStorageDb->pathExists(${$self->{oTargetPath}}{$strTarget}))
{
next;
}
}
# Build a manifest of the path to check for existing files
- my $hTargetManifest = $self->{oFile}->manifest(PATH_DB_ABSOLUTE, ${$self->{oTargetPath}}{$strTarget});
+ my $hTargetManifest = $oStorageDb->manifest(${$self->{oTargetPath}}{$strTarget});
for my $strName (keys(%{$hTargetManifest}))
{
@@ -605,14 +584,14 @@ sub clean
&log(INFO, "remove invalid files/paths/links from ${$self->{oTargetPath}}{$strTarget}");
# OK for the special tablespace path to not exist yet - it will be created later
- if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, ${$self->{oTargetPath}}{$strTarget}) &&
+ if (!$oStorageDb->pathExists(${$self->{oTargetPath}}{$strTarget}) &&
$oManifest->isTargetTablespace($strTarget))
{
next;
}
# Load path manifest so it can be compared to deleted files/paths/links that are not in the backup
- my $hTargetManifest = $self->{oFile}->manifest(PATH_DB_ABSOLUTE, ${$self->{oTargetPath}}{$strTarget});
+ my $hTargetManifest = $oStorageDb->manifest(${$self->{oTargetPath}}{$strTarget});
# If the target is a file it doesn't matter whether it already exists or not.
if ($oManifest->isTargetFile($strTarget))
@@ -658,7 +637,7 @@ sub clean
{
&log(DETAIL, "set ownership ${strUser}:${strGroup} on ${strOsFile}");
- $self->{oFile}->owner(PATH_DB_ABSOLUTE, $strOsFile, $strUser, $strGroup);
+ $oStorageDb->owner($strOsFile, $strUser, $strGroup);
}
# If a link does not have the same destination, then delete it (it will be recreated later)
@@ -668,7 +647,7 @@ sub clean
$hTargetManifest->{$strName}{link_destination})
{
&log(DETAIL, "remove link ${strOsFile} - destination changed");
- fileRemove($strOsFile);
+ $oStorageDb->remove($strOsFile);
}
}
# Else if file/path mode does not match, fix it
@@ -710,10 +689,10 @@ sub clean
else
{
&log(DETAIL, "remove ${strType} ${strOsFile}");
- fileRemove($strOsFile);
+ $oStorageDb->remove($strOsFile);
# Removing a file can be expensive so send protocol keep alive
- $self->{oProtocol}->keepAlive();
+ protocolKeepAlive();
$oRemoveHash{$strSection} += 1;
}
@@ -786,6 +765,9 @@ sub build
{name => 'oManifest'}
);
+ # Db storage
+ my $oStorageDb = storageDb();
+
# Create target paths (except for MANIFEST_TARGET_PGDATA because that must already exist)
foreach my $strTarget ($oManifest->keys(MANIFEST_SECTION_BACKUP_TARGET))
{
@@ -798,10 +780,10 @@ sub build
$strPath = dirname($strPath);
}
- if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strPath))
+ if (!$oStorageDb->pathExists($strPath))
{
- $self->{oFile}->pathCreate(PATH_DB_ABSOLUTE, $strPath,
- $oManifest->get(MANIFEST_SECTION_TARGET_PATH, $strTarget, MANIFEST_SUBKEY_MODE));
+ $oStorageDb->pathCreate(
+ $strPath, {strMode => $oManifest->get(MANIFEST_SECTION_TARGET_PATH, $strTarget, MANIFEST_SUBKEY_MODE)});
# Set ownership (??? this could be done better inside the file functions)
my $strUser = $oManifest->get(MANIFEST_SECTION_TARGET_PATH, $strTarget, MANIFEST_SUBKEY_USER);
@@ -809,7 +791,7 @@ sub build
if ($strUser ne getpwuid($<) || $strGroup ne getgrgid($())
{
- $self->{oFile}->owner(PATH_DB_ABSOLUTE, $strPath, $strUser, $strGroup);
+ $oStorageDb->owner($strPath, $strUser, $strGroup);
}
}
}
@@ -850,13 +832,13 @@ sub build
# If the path/link does not already exist then create it. The clean() method should have determined if the
# permissions, destinations, etc. are correct
- if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strDbPath))
+ if (!$oStorageDb->pathExists($strDbPath) && !$oStorageDb->exists($strDbPath))
{
# Create a path
if ($strSection eq &MANIFEST_SECTION_TARGET_PATH)
{
- $self->{oFile}->pathCreate(PATH_DB_ABSOLUTE, $strDbPath,
- $oManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODE));
+ $oStorageDb->pathCreate(
+ $strDbPath, {strMode => $oManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODE)});
}
# Else create a link
else
@@ -867,8 +849,10 @@ sub build
# In order to create relative links they must be converted to absolute links and then made relative
# again by linkCreate(). It's possible to modify linkCreate() to accept relative paths but that could
# have an impact elsewhere and doesn't seem worth it.
- $self->{oFile}->linkCreate(PATH_DB_ABSOLUTE, pathAbsolute(dirname($strDbPath), $strDestination),
- PATH_DB_ABSOLUTE, $strDbPath, undef, (index($strDestination, '/') != 0));
+ $oStorageDb->linkCreate(
+ $oStorageDb->pathAbsolute(
+ dirname($strDbPath), $strDestination), $strDbPath,
+ {bRelative => (index($strDestination, '/') != 0, bIgnoreExists => true)});
}
# Set ownership (??? this could be done better inside the file functions)
@@ -877,7 +861,7 @@ sub build
if ($strUser ne getpwuid($<) || $strGroup ne getgrgid($())
{
- $self->{oFile}->owner(PATH_DB_ABSOLUTE, $strDbPath, $strUser, $strGroup);
+ $oStorageDb->owner($strDbPath, $strUser, $strGroup);
}
}
@@ -913,7 +897,7 @@ sub recovery
my $strRecoveryConf = $self->{strDbClusterPath} . '/' . DB_FILE_RECOVERYCONF;
# See if recovery.conf already exists
- my $bRecoveryConfExists = $self->{oFile}->exists(PATH_DB_ABSOLUTE, $strRecoveryConf);
+ my $bRecoveryConfExists = storageDb()->exists($strRecoveryConf);
# If RECOVERY_TYPE_PRESERVE then warn if recovery.conf does not exist and return
if (optionTest(OPTION_TYPE, RECOVERY_TYPE_PRESERVE))
@@ -928,7 +912,7 @@ sub recovery
# In all other cases the old recovery.conf should be removed if it exists
if ($bRecoveryConfExists)
{
- $self->{oFile}->remove(PATH_DB_ABSOLUTE, $strRecoveryConf);
+ storageDb()->remove($strRecoveryConf);
}
# If RECOVERY_TYPE_NONE then return
@@ -1040,13 +1024,16 @@ sub process
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam (__PACKAGE__ . '->process');
- if (!fileExists($self->{strDbClusterPath}))
+ # Db storage
+ my $oStorageDb = storageDb();
+
+ if (!$oStorageDb->pathExists($self->{strDbClusterPath}))
{
confess &log(ERROR, "\$PGDATA directory $self->{strDbClusterPath} does not exist");
}
# Make sure that Postgres is not running
- if ($self->{oFile}->exists(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . DB_FILE_POSTMASTERPID))
+ if ($oStorageDb->exists($self->{strDbClusterPath} . '/' . DB_FILE_POSTMASTERPID))
{
confess &log(ERROR,
"unable to restore while PostgreSQL is running\n" .
@@ -1057,8 +1044,8 @@ sub process
# If the restore will be destructive attempt to verify that $PGDATA is valid
if ((optionGet(OPTION_DELTA) || optionGet(OPTION_FORCE)) &&
- !($self->{oFile}->exists(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . DB_FILE_PGVERSION) ||
- $self->{oFile}->exists(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST)))
+ !($oStorageDb->exists($self->{strDbClusterPath} . '/' . DB_FILE_PGVERSION) ||
+ $oStorageDb->exists($self->{strDbClusterPath} . '/' . FILE_MANIFEST)))
{
&log(WARN, '--delta or --force specified but unable to find \'' . DB_FILE_PGVERSION . '\' or \'' . FILE_MANIFEST .
'\' in \'' . $self->{strDbClusterPath} . '\' to confirm that this is a valid $PGDATA directory.' .
@@ -1070,12 +1057,13 @@ sub process
}
# Copy backup info, load it, then delete
- $self->{oFile}->copy(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO,
- PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO);
+ $oStorageDb->copy(
+ storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO, {bProtocolCompress => true}),
+ $self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO);
- my $oBackupInfo = new pgBackRest::Backup::Info($self->{strDbClusterPath}, false);
+ my $oBackupInfo = new pgBackRest::Backup::Info($self->{strDbClusterPath}, false, undef, {oStorage => storageDb()});
- $self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO, undef, false, true);
+ $oStorageDb->remove($self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO);
# If set to restore is latest then get the actual set
if ($self->{strBackupSet} eq OPTION_DEFAULT_RESTORE_SET)
@@ -1104,9 +1092,10 @@ sub process
# Delete pg_control file. This will be copied from the backup at the very end to prevent a partially restored database
# from being started by PostgreSQL.
- $self->{oFile}->remove(PATH_DB_ABSOLUTE, $oManifest->dbPathGet(
- $oManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH),
- MANIFEST_FILE_PGCONTROL), true, true);
+ $oStorageDb->remove(
+ $oManifest->dbPathGet(
+ $oManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH),
+ MANIFEST_FILE_PGCONTROL));
# Clean the restore paths
$self->clean($oManifest);
@@ -1238,7 +1227,7 @@ sub process
# Copy pg_control to a temporary file that will be renamed later
if ($strRepoFile eq MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL)
{
- $strDbFile .= '.' . BACKREST_EXE;
+ $strDbFile .= '.' . STORAGE_TEMP_EXT;
}
# Increment file size
@@ -1272,26 +1261,32 @@ sub process
# A keep-alive is required here because if there are a large number of resumed files that need to be checksummed
# then the remote might timeout while waiting for a command.
- $self->{oProtocol}->keepAlive();
+ protocolKeepAlive();
}
# Create recovery.conf file
$self->recovery($oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION));
# Sync db cluster path
- $self->{oFile}->pathSync(PATH_DB_ABSOLUTE, $self->{strDbClusterPath}, true);
+ $oStorageDb->pathSync($self->{strDbClusterPath}, {bRecurse => true});
- # Copy pg_control last
+ # Move pg_control last
&log(INFO,
'restore ' . $oManifest->dbPathGet(undef, MANIFEST_FILE_PGCONTROL) .
- ' (copied last to ensure aborted restores cannot be started)');
+ ' (performed last to ensure aborted restores cannot be started)');
- $self->{oFile}->move(
- PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . DB_FILE_PGCONTROL . '.' . BACKREST_EXE,
- PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . DB_FILE_PGCONTROL, undef, true);
+ $oStorageDb->move(
+ $self->{strDbClusterPath} . '/' . DB_FILE_PGCONTROL . '.' . STORAGE_TEMP_EXT,
+ $self->{strDbClusterPath} . '/' . DB_FILE_PGCONTROL);
+
+ # Sync to be sure the rename of pg_control is persisted
+ $oStorageDb->pathSync($self->{strDbClusterPath});
# Finally remove the manifest to indicate the restore is complete
- $self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST, undef, false, true);
+ $oStorageDb->remove($self->{strDbClusterPath} . '/' . FILE_MANIFEST);
+
+ # Sync removal of the manifest
+ $oStorageDb->pathSync($self->{strDbClusterPath});
# Return from function and log return values if any
return logDebugReturn($strOperation);
diff --git a/lib/pgBackRest/RestoreFile.pm b/lib/pgBackRest/RestoreFile.pm
index 37a034986..7ee8f8aa2 100644
--- a/lib/pgBackRest/RestoreFile.pm
+++ b/lib/pgBackRest/RestoreFile.pm
@@ -9,18 +9,20 @@ use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
-use Fcntl qw(O_WRONLY O_CREAT O_TRUNC);
use File::Basename qw(dirname);
use File::stat qw(lstat);
use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::Manifest;
-use pgBackRest::Protocol::Common::Common;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Filter::Gzip;
+use pgBackRest::Storage::Filter::Sha;
+use pgBackRest::Storage::Helper;
####################################################################################################################################
# restoreFile
@@ -33,7 +35,6 @@ sub restoreFile
my
(
$strOperation,
- $oFile, # File object
$strDbFile,
$lSize,
$lModificationTime,
@@ -53,7 +54,6 @@ sub restoreFile
logDebugParam
(
__PACKAGE__ . '::restoreFile', \@_,
- {name => 'oFile', trace => true},
{name => 'strDbFile', trace => true},
{name => 'lSize', trace => true},
{name => 'lModificationTime', trace => true},
@@ -71,35 +71,24 @@ sub restoreFile
{name => 'bSourceCompressed', trace => true},
);
- # Copy flag and log message
+ # Does the file need to be copied?
+ my $oStorageDb = storageDb();
my $bCopy = true;
if ($bZero)
{
$bCopy = false;
- # Open the file truncating to zero bytes in case it already exists
- my $hFile = fileOpen($strDbFile, O_WRONLY | O_CREAT | O_TRUNC, $strMode);
+ my $oDestinationFileIo = $oStorageDb->openWrite(
+ $strDbFile, {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lModificationTime});
+ $oDestinationFileIo->open();
# Now truncate to the original size. This will create a sparse file which is very efficient for this use case.
- truncate($hFile, $lSize);
+ truncate($oDestinationFileIo->handle(), $lSize);
- # Sync the file
- $hFile->sync()
- or confess &log(ERROR, "unable to sync ${strDbFile}", ERROR_FILE_SYNC);
-
- # Close the file
- close($hFile)
- or confess &log(ERROR, "unable to close ${strDbFile}", ERROR_FILE_CLOSE);
-
- # Fix the timestamp - not really needed in this case but good for testing
- utime($lModificationTime, $lModificationTime, $strDbFile)
- or confess &log(ERROR, "unable to set time for ${strDbFile}");
-
- # Set file ownership
- $oFile->owner(PATH_DB_ABSOLUTE, $strDbFile, $strUser, $strGroup);
+ $oDestinationFileIo->close();
}
- elsif ($oFile->exists(PATH_DB_ABSOLUTE, $strDbFile))
+ elsif ($oStorageDb->exists($strDbFile))
{
# Perform delta if requested
if ($bDelta)
@@ -118,7 +107,7 @@ sub restoreFile
}
else
{
- my ($strActualChecksum, $lActualSize) = $oFile->hashSize(PATH_DB_ABSOLUTE, $strDbFile);
+ my ($strActualChecksum, $lActualSize) = $oStorageDb->hashSize($strDbFile);
if ($lActualSize == $lSize && ($lSize == 0 || $strActualChecksum eq $strChecksum))
{
@@ -133,25 +122,38 @@ sub restoreFile
}
}
- # Copy the file from the backup to the database
+ # Copy file from repository to database
if ($bCopy)
{
- my ($bCopyResult, $strCopyChecksum, $lCopySize) = $oFile->copy(
- PATH_BACKUP_CLUSTER, (defined($strReference) ? $strReference : $strBackupPath) .
- "/${strRepoFile}" . ($bSourceCompressed ? '.' . $oFile->{strCompressExtension} : ''),
- PATH_DB_ABSOLUTE, $strDbFile,
- $bSourceCompressed,
- undef, undef,
- $lModificationTime, $strMode,
- undef,
- $strUser, $strGroup,
- undef, undef, undef, undef,
- false); # Don't copy via a temp file
+ # Add sha filter
+ my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];
- if ($lCopySize != 0 && $strCopyChecksum ne $strChecksum)
+ # Add compression
+ if ($bSourceCompressed)
{
- confess &log(ERROR, "error restoring ${strDbFile}: actual checksum ${strCopyChecksum} " .
- "does not match expected checksum ${strChecksum}", ERROR_CHECKSUM);
+ unshift(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]});
+ }
+
+ # Open destination file
+ my $oDestinationFileIo = $oStorageDb->openWrite(
+ $strDbFile,
+ {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lModificationTime,
+ rhyFilter => $rhyFilter});
+
+ # Copy file
+ storageRepo()->copy(
+ storageRepo()->openRead(
+ STORAGE_REPO_BACKUP . qw(/) . (defined($strReference) ? $strReference : $strBackupPath) .
+ "/${strRepoFile}" . ($bSourceCompressed ? qw{.} . COMPRESS_EXT : ''),
+ {bProtocolCompress => !$bSourceCompressed && $lSize != 0}),
+ $oDestinationFileIo);
+
+ # Validate checksum
+ if ($oDestinationFileIo->result(COMMON_IO_HANDLE) != 0 && $oDestinationFileIo->result(STORAGE_FILTER_SHA) ne $strChecksum)
+ {
+ confess &log(ERROR,
+ "error restoring ${strDbFile}: actual checksum '" . $oDestinationFileIo->digest() .
+ "' does not match expected checksum ${strChecksum}", ERROR_CHECKSUM);
}
}
diff --git a/lib/pgBackRest/Stanza.pm b/lib/pgBackRest/Stanza.pm
index b8c95ef89..239735800 100644
--- a/lib/pgBackRest/Stanza.pm
+++ b/lib/pgBackRest/Stanza.pm
@@ -20,17 +20,16 @@ use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Info;
use pgBackRest::Db;
use pgBackRest::DbVersion;
-use pgBackRest::File;
-use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
-use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
+use pgBackRest::Protocol::Storage::Helper;
####################################################################################################################################
# Global variables
####################################################################################################################################
-my $strStanzaCreateErrorMsg = "not empty\n" .
- "HINT: Use --force to force the stanza data to be created.";
+my $strHintForce = "\nHINT: use stanza-create --force to force the stanza data to be created.";
+my $strInfoMissing = " information missing";
+my $strStanzaCreateErrorMsg = "not empty" . $strHintForce;
####################################################################################################################################
# CONSTRUCTOR
@@ -106,51 +105,63 @@ sub stanzaCreate
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->stanzaCreate');
- # Initialize default file object with protocol set to NONE meaning strictly local
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(NONE)
- );
-
# Get the parent paths (create if not exist)
- my $strParentPathArchive = $self->parentPathGet($oFile, PATH_BACKUP_ARCHIVE);
- my $strParentPathBackup = $self->parentPathGet($oFile, PATH_BACKUP_CLUSTER);
+ my $strParentPathArchive = $self->parentPathGet(STORAGE_REPO_ARCHIVE);
+ my $strParentPathBackup = $self->parentPathGet(STORAGE_REPO_BACKUP);
# Get a listing of files in the directory, ignoring if any are missing
- my @stryFileListArchive = fileList($strParentPathArchive, {bIgnoreMissing => true});
- my @stryFileListBackup = fileList($strParentPathBackup, {bIgnoreMissing => true});
+ my @stryFileListArchive = storageRepo()->list($strParentPathArchive, {bIgnoreMissing => true});
+ my @stryFileListBackup = storageRepo()->list($strParentPathBackup, {bIgnoreMissing => true});
- # If force not used and at least one directory is not empty, then check to see if the info files exist
- if (!optionGet(OPTION_FORCE) && (@stryFileListArchive || @stryFileListBackup))
+ # If force not used, then if files exist force should be required since create must have already occurred and reissuing a create
+ # needs to be a consciuos effort to rewrite the files
+ if (!optionGet(OPTION_FORCE))
{
- my $strBackupInfoFile = &FILE_BACKUP_INFO;
- my $strArchiveInfoFile = &ARCHIVE_INFO_FILE;
-
- # If either info file is not in the file list, then something exists in the directories so need to use force option
- if (@stryFileListBackup && !grep(/^$strBackupInfoFile/i, @stryFileListBackup)
- || @stryFileListArchive && !grep(/^$strArchiveInfoFile/i, @stryFileListArchive))
+ # At least one directory is not empty, then check to see if the info files exist
+ if (@stryFileListArchive || @stryFileListBackup)
{
- confess &log(ERROR,
- (@stryFileListBackup ? 'backup directory ' : '') .
- ((@stryFileListBackup && @stryFileListArchive) ? 'and/or ' : '') .
- (@stryFileListArchive ? 'archive directory ' : '') .
- $strStanzaCreateErrorMsg, ERROR_PATH_NOT_EMPTY);
+ my $strBackupInfoFile = &FILE_BACKUP_INFO;
+ my $strArchiveInfoFile = &ARCHIVE_INFO_FILE;
+
+ my $bBackupInfoFileExists = grep(/^$strBackupInfoFile$/i, @stryFileListBackup);
+ my $bArchiveInfoFileExists = grep(/^$strArchiveInfoFile$/i, @stryFileListArchive);
+
+ # If the info file exists in one directory but is missing from the other directory then there is clearly a mismatch
+ # which requires force option
+ if (!$bArchiveInfoFileExists && $bBackupInfoFileExists)
+ {
+ confess &log(ERROR, 'archive' . $strInfoMissing . $strHintForce, ERROR_FILE_MISSING);
+ }
+ elsif (!$bBackupInfoFileExists && $bArchiveInfoFileExists)
+ {
+ confess &log(ERROR, 'backup' . $strInfoMissing . $strHintForce, ERROR_FILE_MISSING);
+ }
+ # If we get here then either both exist or neither exist so if neither file exists then something still exists in the
+ # directories since one or both of them are not empty so need to use force option
+ elsif (!$bArchiveInfoFileExists)
+ {
+ confess &log(ERROR,
+ (@stryFileListBackup ? 'backup directory ' : '') .
+ ((@stryFileListBackup && @stryFileListArchive) ? 'and/or ' : '') .
+ (@stryFileListArchive ? 'archive directory ' : '') .
+ $strStanzaCreateErrorMsg, ERROR_PATH_NOT_EMPTY);
+ }
}
}
- # Create the archive.info file and local variables
+ # Instantiate the info objects. Throws an error and aborts if force not used and an error occurs during instantiation.
+ my $oArchiveInfo = $self->infoObject(STORAGE_REPO_ARCHIVE, $strParentPathArchive, {bRequired => false, bIgnoreMissing => true});
+ my $oBackupInfo = $self->infoObject(STORAGE_REPO_BACKUP, $strParentPathBackup, {bRequired => false, bIgnoreMissing => true});
+
+ # Create the archive info object
my ($iResult, $strResultMessage) =
- $self->infoFileCreate((new pgBackRest::Archive::ArchiveInfo($strParentPathArchive, false)), $oFile,
- PATH_BACKUP_ARCHIVE, $strParentPathArchive, \@stryFileListArchive);
+ $self->infoFileCreate($oArchiveInfo, STORAGE_REPO_ARCHIVE, $strParentPathArchive, \@stryFileListArchive);
if ($iResult == 0)
{
# Create the backup.info file
($iResult, $strResultMessage) =
- $self->infoFileCreate((new pgBackRest::Backup::Info($strParentPathBackup, false, false)), $oFile,
- PATH_BACKUP_CLUSTER, $strParentPathBackup, \@stryFileListBackup);
+ $self->infoFileCreate($oBackupInfo, STORAGE_REPO_BACKUP, $strParentPathBackup, \@stryFileListBackup);
}
if ($iResult != 0)
@@ -180,47 +191,29 @@ sub stanzaUpgrade
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->stanzaUpgrade');
- # Initialize default file object with protocol set to NONE meaning strictly local
- my $oFile = new pgBackRest::File
- (
- optionGet(OPTION_STANZA),
- optionGet(OPTION_REPO_PATH),
- protocolGet(NONE)
- );
-
# Get the archive info and backup info files; if either does not exist an error will be thrown
- my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE));
- my $oBackupInfo = new pgBackRest::Backup::Info($oFile->pathGet(PATH_BACKUP_CLUSTER));
+ my $oArchiveInfo = $self->infoObject(STORAGE_REPO_ARCHIVE, storageRepo()->pathGet(STORAGE_REPO_ARCHIVE));
+ my $oBackupInfo = $self->infoObject(STORAGE_REPO_BACKUP, storageRepo()->pathGet(STORAGE_REPO_BACKUP));
my $bBackupUpgraded = false;
my $bArchiveUpgraded = false;
# If the DB section does not match, then upgrade
- if ($self->upgradeCheck($oBackupInfo, PATH_BACKUP_CLUSTER, ERROR_BACKUP_MISMATCH))
+ if ($self->upgradeCheck($oBackupInfo, STORAGE_REPO_BACKUP, ERROR_BACKUP_MISMATCH))
{
- # Determine if it is necessary to reconstruct the file
- my ($bReconstruct, $strWarningMsgArchive) =
- $self->reconstructCheck($oBackupInfo, PATH_BACKUP_CLUSTER, $oFile, $oFile->pathGet(PATH_BACKUP_CLUSTER));
-
- # If reconstruction was required then save the reconstructed file
- if ($bReconstruct)
- {
- $oBackupInfo->save();
- $bBackupUpgraded = true;
- }
+ # Reconstruct the file and save it
+ my ($bReconstruct, $strWarningMsgArchive) = $oBackupInfo->reconstruct(false, false, $self->{oDb}{strDbVersion},
+ $self->{oDb}{ullDbSysId}, $self->{oDb}{iControlVersion}, $self->{oDb}{iCatalogVersion});
+ $oBackupInfo->save();
+ $bBackupUpgraded = true;
}
- if ($self->upgradeCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, ERROR_ARCHIVE_MISMATCH))
+ if ($self->upgradeCheck($oArchiveInfo, STORAGE_REPO_ARCHIVE, ERROR_ARCHIVE_MISMATCH))
{
- # Determine if it is necessary to reconstruct the file
- my ($bReconstruct, $strWarningMsgArchive) =
- $self->reconstructCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, $oFile, $oFile->pathGet(PATH_BACKUP_ARCHIVE));
-
- # If reconstruction was required then save the reconstructed file
- if ($bReconstruct)
- {
- $oArchiveInfo->save();
- $bArchiveUpgraded = true;
- }
+ # Reconstruct the file and save it
+ my ($bReconstruct, $strWarningMsgArchive) = $oArchiveInfo->reconstruct($self->{oDb}{strDbVersion},
+ $self->{oDb}{ullDbSysId});
+ $oArchiveInfo->save();
+ $bArchiveUpgraded = true;
}
# If neither file needed upgrading then provide informational message that an upgrade was not necessary
@@ -250,23 +243,21 @@ sub parentPathGet
my
(
$strOperation,
- $oFile,
$strPathType,
) =
logDebugParam
(
__PACKAGE__ . '->parentPathGet', \@_,
- {name => 'oFile', trace => true},
{name => 'strPathType', trace => true},
);
- my $strParentPath = $oFile->pathGet($strPathType);
+ my $strParentPath = storageRepo()->pathGet($strPathType);
# If the info path does not exist, create it
- if (!fileExists($strParentPath))
+ if (!storageRepo()->pathExists($strParentPath))
{
# Create the cluster repo path
- $oFile->pathCreate($strPathType, undef, undef, true, true);
+ storageRepo()->pathCreate($strPathType, {bIgnoreExists => true, bCreateParent => true});
}
# Return from function and log return values if any
@@ -277,6 +268,97 @@ sub parentPathGet
);
}
+####################################################################################################################################
+# infoObject
+#
+# Attempt to load an info object. Ignores missing files if directed. Throws an error and aborts if force not used and an error
+# occurs during loading, else instatiates the object without loading it.
+####################################################################################################################################
+sub infoObject
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathType,
+ $strParentPath,
+ $bRequired,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->infoObject', \@_,
+ {name => 'strPathType'},
+ {name => 'strParentPath'},
+ {name => 'bRequired', optional => true, default => true},
+ {name => 'bIgnoreMissing', optional => true, default => false},
+ );
+
+ my $iResult = 0;
+ my $strResultMessage;
+ my $oInfo;
+
+ # Turn off console logging to control when to display the error
+ logDisable();
+
+ # Instantiate the info object in an eval block to trap errors. If force is not used and an error occurs, throw the error
+ # along with a directive that force will need to be used to attempt to correct the issue
+ eval
+ {
+ # Ignore missing files if directed but if the info or info.copy file exists the exists flag will still be set and data will
+ # attempt to be loaded
+ $oInfo = ($strPathType eq STORAGE_REPO_BACKUP ?
+ new pgBackRest::Backup::Info($strParentPath, false, $bRequired, {bIgnoreMissing => $bIgnoreMissing}) :
+ new pgBackRest::Archive::ArchiveInfo($strParentPath, $bRequired, {bIgnoreMissing => $bIgnoreMissing}));
+
+ # Reset the console logging
+ logEnable();
+ return true;
+ }
+ or do
+ {
+ # Reset console logging and capture error information
+ logEnable();
+ $iResult = exceptionCode($EVAL_ERROR);
+ $strResultMessage = exceptionMessage($EVAL_ERROR->message());
+ };
+
+ if ($iResult != 0)
+ {
+ # If force was not used, and the file is missing, then confess the error with hint to use force if the option is
+ # configurable (force is not configurable for stanza-upgrade so this will always confess errors on stanza-upgrade)
+ # else confess all other errors
+ if ((optionValid(OPTION_FORCE) && !optionGet(OPTION_FORCE)) ||
+ (!optionValid(OPTION_FORCE)))
+ {
+ if ($iResult == ERROR_FILE_MISSING)
+ {
+ confess &log(ERROR, (optionValid(OPTION_FORCE) ? $strResultMessage . $strHintForce : $strResultMessage), $iResult);
+ }
+ else
+ {
+ confess &log(ERROR, $strResultMessage, $iResult);
+ }
+ }
+ # Else instatiate the object without loading it so we can reconstruct and overwrite the invalid files
+ else
+ {
+ $oInfo = ($strPathType eq STORAGE_REPO_BACKUP ?
+ new pgBackRest::Backup::Info($strParentPath, false, false, {bLoad => false}) :
+ new pgBackRest::Archive::ArchiveInfo($strParentPath, false, {bLoad => false}));
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oInfo', value => $oInfo},
+ );
+}
+
####################################################################################################################################
# infoFileCreate
#
@@ -291,7 +373,6 @@ sub infoFileCreate
(
$strOperation,
$oInfo,
- $oFile,
$strPathType,
$strParentPath,
$stryFileList,
@@ -300,7 +381,6 @@ sub infoFileCreate
(
__PACKAGE__ . '->infoFileCreate', \@_,
{name => 'oInfo', trace => true},
- {name => 'oFile', trace => true},
{name => 'strPathType'},
{name => 'strParentPath'},
{name => 'stryFileList'},
@@ -309,14 +389,12 @@ sub infoFileCreate
my $iResult = 0;
my $strResultMessage = undef;
my $strWarningMsgArchive = undef;
- my $bReconstruct = true;
-
# If force was not used and the info file does not exist and the directory is not empty, then error
# This should also be performed by the calling routine before this function is called, so this is just a safety check
if (!optionGet(OPTION_FORCE) && !$oInfo->{bExists} && @$stryFileList)
{
- confess &log(ERROR, ($strPathType eq PATH_BACKUP_CLUSTER ? 'backup directory ' : 'archive directory ') .
+ confess &log(ERROR, ($strPathType eq STORAGE_REPO_BACKUP ? 'backup directory ' : 'archive directory ') .
$strStanzaCreateErrorMsg, ERROR_PATH_NOT_EMPTY);
}
@@ -325,34 +403,36 @@ sub infoFileCreate
eval
{
- ($bReconstruct, $strWarningMsgArchive) = $self->reconstructCheck($oInfo, $strPathType, $oFile, $strParentPath);
-
- if ($oInfo->exists() && $bReconstruct)
+ # Reconstruct the file from the data in the directory if there is any else initialize the file
+ if ($strPathType eq STORAGE_REPO_BACKUP)
{
- # If force was not used and the hashes are different then error
- if (!optionGet(OPTION_FORCE))
- {
- $iResult = ERROR_FILE_INVALID;
- $strResultMessage =
- ($strPathType eq PATH_BACKUP_CLUSTER ? 'backup file ' : 'archive file ') . "invalid\n" .
- 'HINT: use stanza-upgrade if the database has been upgraded or use --force';
- }
+ $oInfo->reconstruct(false, false, $self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, $self->{oDb}{iControlVersion},
+ $self->{oDb}{iCatalogVersion});
+ }
+ # If this is the archive.info reconstruction then catch any warnings
+ else
+ {
+ $strWarningMsgArchive = $oInfo->reconstruct($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId});
}
- if ($iResult == 0)
+ # If the file exists on disk, then check if the reconstructed data is the same as what is on disk
+ if ($oInfo->exists())
{
- # Save the reconstructed file
- if ($bReconstruct)
- {
- $oInfo->save();
- }
+ my $oInfoOnDisk =
+ ($strPathType eq STORAGE_REPO_BACKUP ?
+ new pgBackRest::Backup::Info($strParentPath) : new pgBackRest::Archive::ArchiveInfo($strParentPath));
- # Sync path if requested
- if (optionGet(OPTION_REPO_SYNC))
+ # If the hashes are not the same
+ if ($oInfoOnDisk->hash() ne $oInfo->hash())
{
- $oFile->pathSync(
- PATH_BACKUP_ABSOLUTE,
- defined($oInfo->{strArchiveClusterPath}) ? $oInfo->{strArchiveClusterPath} : $oInfo->{strBackupClusterPath});
+ # If force was not used and the hashes are different then error
+ if (!optionGet(OPTION_FORCE))
+ {
+ $iResult = ERROR_FILE_INVALID;
+ $strResultMessage =
+ ($strPathType eq STORAGE_REPO_BACKUP ? 'backup info file ' : 'archive info file ') . "invalid\n" .
+ 'HINT: use stanza-upgrade if the database has been upgraded or use --force';
+ }
}
}
@@ -368,6 +448,16 @@ sub infoFileCreate
$strResultMessage = exceptionMessage($EVAL_ERROR->message());
};
+ # If we got here without error then save the reconstructed file
+ if ($iResult == 0)
+ {
+ $oInfo->save();
+
+ # Sync path
+ storageRepo()->pathSync(
+ defined($oInfo->{strArchiveClusterPath}) ? $oInfo->{strArchiveClusterPath} : $oInfo->{strBackupClusterPath});
+ }
+
# If a warning was issued, raise it
if (defined($strWarningMsgArchive))
{
@@ -411,73 +501,6 @@ sub dbInfoGet
return logDebugReturn($strOperation);
}
-####################################################################################################################################
-# reconstructCheck
-#
-# Reconstruct the file based on disk data. If the info file already exists, it compares the reconstructed file to the existing file
-# and indicates if reconstruction is required. If the file does not yet exist on disk, it will still indicate reconstruction is
-# needed. The oInfo object contains the reconstructed data and can be saved by the calling routine.
-####################################################################################################################################
-sub reconstructCheck
-{
- my $self = shift;
-
- # Assign function parameters, defaults, and log debug info
- my
- (
- $strOperation,
- $oInfo,
- $strPathType,
- $oFile,
- $strParentPath,
- ) =
- logDebugParam
- (
- __PACKAGE__ . '->reconstructCheck', \@_,
- {name => 'oInfo'},
- {name => 'strPathType'},
- {name => 'oFile'},
- {name => 'strParentPath'},
- );
-
- my $bReconstruct = true;
- my $strWarningMsgArchive = undef;
-
- # Reconstruct the file from the data in the directory if there is any else initialize the file
- if ($strPathType eq PATH_BACKUP_CLUSTER)
- {
- $oInfo->reconstruct(false, false, $self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, $self->{oDb}{iControlVersion},
- $self->{oDb}{iCatalogVersion});
- }
- # If this is the archive.info reconstruction then catch any warnings
- else
- {
- $strWarningMsgArchive = $oInfo->reconstruct($oFile, $self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId});
- }
-
- # If the file exists on disk, then check if the reconstructed data is the same as what is on disk
- if ($oInfo->{bExists})
- {
- my $oInfoOnDisk =
- ($strPathType eq PATH_BACKUP_CLUSTER ? new pgBackRest::Backup::Info($strParentPath)
- : new pgBackRest::Archive::ArchiveInfo($strParentPath));
-
- # If the hashes are the same, then no need to reconstruct the file since it already exists and is valid
- if ($oInfoOnDisk->hash() eq $oInfo->hash())
- {
- $bReconstruct = false;
- }
- }
-
- # Return from function and log return values if any
- return logDebugReturn
- (
- $strOperation,
- {name => 'bReconstruct', value => $bReconstruct},
- {name => 'strWarningMsgArchive', value => $strWarningMsgArchive},
- );
-}
-
####################################################################################################################################
# upgradeCheck
#
@@ -511,7 +534,7 @@ sub upgradeCheck
eval
{
- ($strPathType eq PATH_BACKUP_CLUSTER)
+ ($strPathType eq STORAGE_REPO_BACKUP)
? $oInfo->check($self->{oDb}{strDbVersion}, $self->{oDb}{iControlVersion}, $self->{oDb}{iCatalogVersion},
$self->{oDb}{ullDbSysId}, true)
: $oInfo->check($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, true);
diff --git a/lib/pgBackRest/Storage/Base.pm b/lib/pgBackRest/Storage/Base.pm
new file mode 100644
index 000000000..a0311b152
--- /dev/null
+++ b/lib/pgBackRest/Storage/Base.pm
@@ -0,0 +1,295 @@
+####################################################################################################################################
+# Base Storage Module
+####################################################################################################################################
+package pgBackRest::Storage::Base;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use File::Basename qw(dirname);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Base;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Compress constants
+####################################################################################################################################
+use constant STORAGE_COMPRESS => 'compress';
+ push @EXPORT, qw(STORAGE_COMPRESS);
+use constant STORAGE_DECOMPRESS => 'decompress';
+ push @EXPORT, qw(STORAGE_DECOMPRESS);
+
+####################################################################################################################################
+# Capability constants
+####################################################################################################################################
+use constant STORAGE_CAPABILITY_LINK => 'link';
+ push @EXPORT, qw(STORAGE_CAPABILITY_LINK);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Create the class hash
+ my $self = {};
+ bless $self, $class;
+
+ # Assign function parameters, defaults, and log debug info
+ (
+ my $strOperation,
+ $self->{lBufferMax},
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},
+ );
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# copy - copy a file
+####################################################################################################################################
+sub copy
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xSourceFile,
+ $xDestinationFile,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->copy', \@_,
+ {name => 'xSourceFile', required => false},
+ {name => 'xDestinationFile', required => false},
+ );
+
+ # Was the file copied?
+ my $bResult = false;
+
+ # Is source an IO object or a file expression?
+ my $oSourceFileIo =
+ defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($self->pathGet($xSourceFile))) : undef;
+
+ # Proceed if source file exists
+ if (defined($oSourceFileIo))
+ {
+ # Is destination an IO object or a file expression?
+ my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($self->pathGet($xDestinationFile));
+
+ # Copy the data
+ my $lSizeRead;
+
+ do
+ {
+ # Read data
+ my $tBuffer = '';
+
+ $lSizeRead = $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
+ $oDestinationFileIo->write(\$tBuffer);
+ }
+ while ($lSizeRead != 0);
+
+ # Close files
+ $oSourceFileIo->close();
+ $oDestinationFileIo->close();
+
+ # File was copied
+ $bResult = true;
+ }
+
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bResult', value => $bResult, trace => true},
+ );
+}
+
+####################################################################################################################################
+# get - reads a buffer from storage all at once
+####################################################################################################################################
+sub get
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xFile,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->get', \@_,
+ {name => 'xFile', required => false, trace => true},
+ );
+
+ # Is this an IO object or a file expression?
+ my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile)) : undef;
+
+ # Read only if there is something to read from
+ my $tContent;
+ my $lSize = 0;
+
+ if (defined($oFileIo))
+ {
+ my $lSizeRead;
+
+ do
+ {
+ $lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax});
+ $lSize += $lSizeRead;
+ }
+ while ($lSizeRead != 0);
+
+ # Close the file
+ $oFileIo->close();
+
+ # If nothing was read then set to undef
+ if ($lSize == 0)
+ {
+ $tContent = undef;
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true},
+ );
+}
+
+####################################################################################################################################
+# pathAbsolute - generate an absolute path from an absolute base path and a relative path
+####################################################################################################################################
+sub pathAbsolute
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strBasePath,
+ $strPath
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::pathAbsolute', \@_,
+ {name => 'strBasePath', trace => true},
+ {name => 'strPath', trace => true}
+ );
+
+ # Working variables
+ my $strAbsolutePath;
+
+ # If the path is already absolute
+ if (index($strPath, '/') == 0)
+ {
+ $strAbsolutePath = $strPath;
+ }
+ # Else make it absolute using the base path
+ else
+ {
+ # Make sure the absolute path is really absolute
+ if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1)
+ {
+ confess &log(ERROR, "${strBasePath} is not an absolute path", ERROR_PATH_TYPE);
+ }
+
+ while (index($strPath, '..') == 0)
+ {
+ $strBasePath = dirname($strBasePath);
+ $strPath = substr($strPath, 2);
+
+ if (index($strPath, '/') == 0)
+ {
+ $strPath = substr($strPath, 1);
+ }
+ }
+
+ $strAbsolutePath = "${strBasePath}/${strPath}";
+ }
+
+ # Make sure the result is really an absolute path
+ if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1)
+ {
+ confess &log(ERROR, "result ${strAbsolutePath} was not an absolute path", ERROR_PATH_TYPE);
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'strAbsolutePath', value => $strAbsolutePath, trace => true}
+ );
+}
+
+####################################################################################################################################
+# put - writes a buffer out to storage all at once
+####################################################################################################################################
+sub put
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xFile,
+ $xContent,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->put', \@_,
+ {name => 'xFile', trace => true},
+ {name => 'xContent', required => false, trace => true},
+ );
+
+ # Is this an IO object or a file expression?
+ my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile);
+
+ # Determine size of content
+ my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0;
+
+ # Write only if there is something to write
+ if ($lSize > 0)
+ {
+ $oFileIo->write(ref($xContent) ? $xContent : \$xContent, $lSize);
+ }
+ # Else open the file so a zero length file is created (since file is not opened until first write)
+ else
+ {
+ $oFileIo->open();
+ }
+
+ # Close the file
+ $oFileIo->close();
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'lSize', value => $lSize, trace => true},
+ );
+}
+
+1;
diff --git a/lib/pgBackRest/Storage/Cifs/Driver.pm b/lib/pgBackRest/Storage/Cifs/Driver.pm
new file mode 100644
index 000000000..8136984f5
--- /dev/null
+++ b/lib/pgBackRest/Storage/Cifs/Driver.pm
@@ -0,0 +1,54 @@
+####################################################################################################################################
+# CIFS Storage Driver
+#
+# Implements storage functions for Posix-compliant file systems.
+####################################################################################################################################
+package pgBackRest::Storage::Cifs::Driver;
+use parent 'pgBackRest::Storage::Posix::Driver';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant STORAGE_CIFS_DRIVER => __PACKAGE__;
+ push @EXPORT, qw(STORAGE_CIFS_DRIVER);
+
+####################################################################################################################################
+# pathSync - CIFS does not support path sync so this is a noop
+####################################################################################################################################
+sub pathSync
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathSync', \@_,
+ {name => 'strPath', trace => true},
+ );
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# Getters/Setters
+####################################################################################################################################
+sub capability {false}
+sub className {STORAGE_CIFS_DRIVER}
+
+1;
diff --git a/lib/pgBackRest/Storage/Filter/Gzip.pm b/lib/pgBackRest/Storage/Filter/Gzip.pm
new file mode 100644
index 000000000..fe574a45f
--- /dev/null
+++ b/lib/pgBackRest/Storage/Filter/Gzip.pm
@@ -0,0 +1,263 @@
+####################################################################################################################################
+# GZIP Filter
+####################################################################################################################################
+package pgBackRest::Storage::Filter::Gzip;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Compress::Raw::Zlib qw(WANT_GZIP MAX_WBITS Z_OK Z_BUF_ERROR Z_DATA_ERROR Z_STREAM_END);
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Base;
+use pgBackRest::Common::Log;
+use pgBackRest::Storage::Base;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant STORAGE_FILTER_GZIP => __PACKAGE__;
+ push @EXPORT, qw(STORAGE_FILTER_GZIP);
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $self,
+ $bWantGzip,
+ $strCompressType,
+ $iLevel,
+ $lCompressBufferMax,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'self', trace => true},
+ {name => 'bWantGzip', optional => true, default => true, trace => true},
+ {name => 'strCompressType', optional => true, default => STORAGE_COMPRESS, trace => true},
+ {name => 'iLevel', optional => true, default => 6, trace => true},
+ {name => 'lCompressBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},
+ );
+
+ # Bless with new class
+ @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+ bless $self, $class;
+
+ # Set variables
+ $self->{bWantGzip} = $bWantGzip;
+ $self->{iLevel} = $iLevel;
+ $self->{lCompressBufferMax} = $lCompressBufferMax;
+ $self->{strCompressType} = $strCompressType;
+
+ # Set read/write
+ $self->{bWrite} = false;
+
+ # Create the zlib object
+ my $iZLibStatus;
+
+ if ($self->{strCompressType} eq STORAGE_COMPRESS)
+ {
+ ($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Deflate(
+ WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Level => $self->{iLevel},
+ Bufsize => $self->{lCompressBufferMax}, AppendOutput => 1);
+
+ $self->{tCompressedBuffer} = undef;
+ }
+ else
+ {
+ ($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Inflate(
+ WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Bufsize => $self->{lCompressBufferMax},
+ LimitOutput => 1, AppendOutput => 1);
+
+ $self->{tUncompressedBuffer} = undef;
+ $self->{lUncompressedBufferSize} = 0;
+ }
+
+ $self->errorCheck($iZLibStatus);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# errorCheck - check status code for errors
+####################################################################################################################################
+sub errorCheck
+{
+ my $self = shift;
+ my $iZLibStatus = shift;
+
+ if (!($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR))
+ {
+ logErrorResult(
+ $self->{bWrite} ? ERROR_FILE_WRITE : ERROR_FILE_READ,
+ 'unable to ' . ($self->{strCompressType} eq STORAGE_COMPRESS ? 'deflate' : 'inflate') . " '$self->{strName}'",
+ $self->{oZLib}->msg());
+ }
+
+ return Z_OK;
+}
+
+####################################################################################################################################
+# read - compress/decompress data
+####################################################################################################################################
+sub read
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+ my $iSize = shift;
+
+ if ($self->{strCompressType} eq STORAGE_COMPRESS)
+ {
+ my $lSizeBegin = defined($$rtBuffer) ? length($$rtBuffer) : 0;
+ my $lUncompressedSize;
+ my $lCompressedSize;
+
+ do
+ {
+ my $tUncompressedBuffer;
+ $lUncompressedSize = $self->SUPER::read(\$tUncompressedBuffer, $iSize);
+
+ if ($lUncompressedSize > 0)
+ {
+ $self->errorCheck($self->{oZLib}->deflate($tUncompressedBuffer, $$rtBuffer));
+ }
+ else
+ {
+ $self->errorCheck($self->{oZLib}->flush($$rtBuffer));
+ }
+
+ $lCompressedSize = length($$rtBuffer) - $lSizeBegin;
+ }
+ while ($lUncompressedSize > 0 && $lCompressedSize < $iSize);
+
+ # Return the actual size read
+ return $lCompressedSize;
+ }
+ else
+ {
+ # If the local buffer size is not large enough to satisfy the request and there is still data to decompress
+ if ($self->{lUncompressedBufferSize} < $iSize)
+ {
+ while ($self->{lUncompressedBufferSize} < $iSize)
+ {
+ if (!defined($self->{tCompressedBuffer}) || length($self->{tCompressedBuffer}) == 0)
+ {
+ $self->SUPER::read(\$self->{tCompressedBuffer}, $self->{lCompressBufferMax});
+ }
+
+ my $iZLibStatus = $self->{oZLib}->inflate($self->{tCompressedBuffer}, $self->{tUncompressedBuffer});
+ $self->{lUncompressedBufferSize} = length($self->{tUncompressedBuffer});
+
+ last if $iZLibStatus == Z_STREAM_END;
+
+ $self->errorCheck($iZLibStatus);
+ }
+ }
+
+ # Actual size is the lesser of the local buffer size or requested size - if the local buffer is smaller than the requested size
+ # it means that there was nothing more to be read.
+ my $iActualSize = $self->{lUncompressedBufferSize} < $iSize ? $self->{lUncompressedBufferSize} : $iSize;
+
+ # Append the to the request buffer
+ $$rtBuffer .= substr($self->{tUncompressedBuffer}, 0, $iActualSize);
+
+ # Truncate local buffer
+ $self->{tUncompressedBuffer} = substr($self->{tUncompressedBuffer}, $iActualSize);
+ $self->{lUncompressedBufferSize} -= $iActualSize;
+
+ # Return the actual size read
+ return $iActualSize;
+ }
+}
+
+####################################################################################################################################
+# write - compress/decompress data
+####################################################################################################################################
+sub write
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+
+ $self->{bWrite} = true;
+
+ if ($self->{strCompressType} eq STORAGE_COMPRESS)
+ {
+ # Compress the data
+ $self->errorCheck($self->{oZLib}->deflate($$rtBuffer, $self->{tCompressedBuffer}));
+
+ # Only write when buffer is full
+ if (defined($self->{tCompressedBuffer}) && length($self->{tCompressedBuffer}) > $self->{lCompressBufferMax})
+ {
+ $self->SUPER::write(\$self->{tCompressedBuffer});
+ $self->{tCompressedBuffer} = undef;
+ }
+ }
+ else
+ {
+ my $tCompressedBuffer = $$rtBuffer;
+
+ while (length($tCompressedBuffer) > 0)
+ {
+ my $tUncompressedBuffer;
+
+ my $iZLibStatus = $self->{oZLib}->inflate($tCompressedBuffer, $tUncompressedBuffer);
+ $self->SUPER::write(\$tUncompressedBuffer);
+
+ last if $iZLibStatus == Z_STREAM_END;
+
+ $self->errorCheck($iZLibStatus);
+ }
+ }
+
+ # Return bytes written
+ return length($$rtBuffer);
+}
+
+####################################################################################################################################
+# close - close the file
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ if (defined($self->{oZLib}))
+ {
+ # Flush the write buffer
+ if ($self->{bWrite})
+ {
+ if ($self->{strCompressType} eq STORAGE_COMPRESS)
+ {
+ # Flush out last compressed bytes
+ $self->errorCheck($self->{oZLib}->flush($self->{tCompressedBuffer}));
+
+ # Write last compressed bytes
+ $self->SUPER::write(\$self->{tCompressedBuffer});
+ }
+ }
+
+ undef($self->{oZLib});
+
+ # Close io
+ return $self->SUPER::close();
+ }
+}
+
+1;
diff --git a/lib/pgBackRest/Storage/Filter/Sha.pm b/lib/pgBackRest/Storage/Filter/Sha.pm
new file mode 100644
index 000000000..71d429d0c
--- /dev/null
+++ b/lib/pgBackRest/Storage/Filter/Sha.pm
@@ -0,0 +1,123 @@
+####################################################################################################################################
+# SHA Filter
+####################################################################################################################################
+package pgBackRest::Storage::Filter::Sha;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant STORAGE_FILTER_SHA => __PACKAGE__;
+ push @EXPORT, qw(STORAGE_FILTER_SHA);
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $self,
+ $strAlgorithm,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'self', trace => true},
+ {name => 'strAlgorithm', optional => true, default => 'sha1', trace => true},
+ );
+
+ # Bless with new class
+ @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA)
+ bless $self, $class;
+
+ # Set variables
+ $self->{strAlgorithm} = $strAlgorithm;
+
+ # Create SHA object
+ $self->{oSha} = Digest::SHA->new($self->{strAlgorithm});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# read - calculate sha digest
+####################################################################################################################################
+sub read
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+ my $iSize = shift;
+
+ # Call the io method
+ my $tShaBuffer;
+ my $iActualSize = $self->SUPER::read(\$tShaBuffer, $iSize);
+
+ # Calculate sha for the returned buffer
+ if ($iActualSize > 0)
+ {
+ $self->{oSha}->add($tShaBuffer);
+ $$rtBuffer .= $tShaBuffer;
+ }
+
+ # Return the actual size read
+ return $iActualSize;
+}
+
+####################################################################################################################################
+# write - calculate sha digest
+####################################################################################################################################
+sub write
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+
+ # Calculate sha for the buffer
+ $self->{oSha}->add($$rtBuffer);
+
+ # Call the io method
+ return $self->SUPER::write($rtBuffer);
+}
+
+####################################################################################################################################
+# close - close the file
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ if (defined($self->{oSha}))
+ {
+ # Set result
+ $self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->hexdigest());
+
+ # Delete the sha object
+ delete($self->{oSha});
+
+ # Close io
+ return $self->SUPER::close();
+ }
+}
+
+1;
diff --git a/lib/pgBackRest/Storage/Helper.pm b/lib/pgBackRest/Storage/Helper.pm
new file mode 100644
index 000000000..9f89ef92d
--- /dev/null
+++ b/lib/pgBackRest/Storage/Helper.pm
@@ -0,0 +1,130 @@
+####################################################################################################################################
+# Local Storage Helper
+####################################################################################################################################
+package pgBackRest::Storage::Helper;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use File::Basename qw(basename);
+
+use pgBackRest::Common::Log;
+use pgBackRest::Config::Config;
+use pgBackRest::Storage::Posix::Driver;
+use pgBackRest::Storage::Local;
+use pgBackRest::Version;
+
+####################################################################################################################################
+# Storage constants
+####################################################################################################################################
+use constant STORAGE_LOCAL => '';
+ push @EXPORT, qw(STORAGE_LOCAL);
+
+use constant STORAGE_SPOOL => '';
+ push @EXPORT, qw(STORAGE_SPOOL);
+use constant STORAGE_SPOOL_ARCHIVE_OUT => '';
+ push @EXPORT, qw(STORAGE_SPOOL_ARCHIVE_OUT);
+
+####################################################################################################################################
+# Compression extension
+####################################################################################################################################
+use constant COMPRESS_EXT => 'gz';
+ push @EXPORT, qw(COMPRESS_EXT);
+
+####################################################################################################################################
+# Temp file extension
+####################################################################################################################################
+use constant STORAGE_TEMP_EXT => BACKREST_EXE . '.tmp';
+ push @EXPORT, qw(STORAGE_TEMP_EXT);
+
+####################################################################################################################################
+# Cache storage so it can be retrieved quickly
+####################################################################################################################################
+my $hStorage;
+
+####################################################################################################################################
+# storageLocal - get local storage
+#
+# Local storage is generally read-only (except for locking) and can never reference a remote path. Used for adhoc activities like
+# reading pgbackrest.conf.
+####################################################################################################################################
+sub storageLocal
+{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::storageLocal', \@_,
+ {name => 'strPath', default => '/', trace => true},
+ );
+
+ # Create storage if not defined
+ if (!defined($hStorage->{&STORAGE_LOCAL}{$strPath}))
+ {
+ # Create local storage
+ $hStorage->{&STORAGE_LOCAL}{$strPath} = new pgBackRest::Storage::Local(
+ $strPath, new pgBackRest::Storage::Posix::Driver(),
+ {strTempExtension => STORAGE_TEMP_EXT,
+ lBufferMax => optionValid(OPTION_BUFFER_SIZE, false) ? optionGet(OPTION_BUFFER_SIZE, false) : undef});
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oStorageLocal', value => $hStorage->{&STORAGE_LOCAL}{$strPath}, trace => true},
+ );
+}
+
+push @EXPORT, qw(storageLocal);
+
+####################################################################################################################################
+# storageSpool - get spool storage
+####################################################################################################################################
+sub storageSpool
+{
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strStanza,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::storageSpool', \@_,
+ {name => 'strStanza', default => optionGet(OPTION_STANZA), trace => true},
+ );
+
+ # Create storage if not defined
+ if (!defined($hStorage->{&STORAGE_SPOOL}{$strStanza}))
+ {
+ # Path rules
+ my $hRule =
+ {
+ &STORAGE_SPOOL_ARCHIVE_OUT => "archive/${strStanza}/out",
+ };
+
+ # Create local storage
+ $hStorage->{&STORAGE_SPOOL}{$strStanza} = new pgBackRest::Storage::Local(
+ optionGet(OPTION_SPOOL_PATH), new pgBackRest::Storage::Posix::Driver(),
+ {hRule => $hRule, strTempExtension => STORAGE_TEMP_EXT, lBufferMax => optionGet(OPTION_BUFFER_SIZE)});
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oStorageSpool', value => $hStorage->{&STORAGE_SPOOL}{$strStanza}, trace => true},
+ );
+}
+
+push @EXPORT, qw(storageSpool);
+
+1;
diff --git a/lib/pgBackRest/Storage/Local.pm b/lib/pgBackRest/Storage/Local.pm
new file mode 100644
index 000000000..d18d5ac7b
--- /dev/null
+++ b/lib/pgBackRest/Storage/Local.pm
@@ -0,0 +1,773 @@
+####################################################################################################################################
+# Local Storage
+#
+# Implements storage functionality using drivers.
+####################################################################################################################################
+package pgBackRest::Storage::Local;
+use parent 'pgBackRest::Storage::Base';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use File::Basename qw(dirname);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::String;
+use pgBackRest::Storage::Filter::Sha;
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathBase,
+ $oDriver,
+ $hRule,
+ $bAllowTemp,
+ $strTempExtension,
+ $strDefaultPathMode,
+ $strDefaultFileMode,
+ $lBufferMax,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'strPathBase'},
+ {name => 'oDriver'},
+ {name => 'hRule', optional => true},
+ {name => 'bAllowTemp', optional => true, default => true},
+ {name => 'strTempExtension', optional => true, default => 'tmp'},
+ {name => 'strDefaultPathMode', optional => true, default => '0750'},
+ {name => 'strDefaultFileMode', optional => true, default => '0640'},
+ {name => 'lBufferMax', optional => true},
+ );
+
+ # Create class
+ my $self = $class->SUPER::new({lBufferMax => $lBufferMax});
+ bless $self, $class;
+
+ $self->{strPathBase} = $strPathBase;
+ $self->{oDriver} = $oDriver;
+ $self->{hRule} = $hRule;
+ $self->{bAllowTemp} = $bAllowTemp;
+ $self->{strTempExtension} = $strTempExtension;
+ $self->{strDefaultPathMode} = $strDefaultPathMode;
+ $self->{strDefaultFileMode} = $strDefaultFileMode;
+
+ # Set temp extension in driver
+ $self->driver()->tempExtensionSet($self->{strTempExtension}) if $self->driver()->can('tempExtensionSet');
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self}
+ );
+}
+
+####################################################################################################################################
+# exists - check if file exists
+####################################################################################################################################
+sub exists
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFileExp,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->exists', \@_,
+ {name => 'strFileExp'},
+ );
+
+ # Check exists
+ my $bExists = $self->driver()->exists($self->pathGet($strFileExp));
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bExists', value => $bExists}
+ );
+}
+
+####################################################################################################################################
+# hashSize - calculate sha1 hash and size of file
+####################################################################################################################################
+sub hashSize
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xFileExp,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->hashSize', \@_,
+ {name => 'xFileExp'},
+ );
+
+ # Set operation variables
+ my $strHash;
+ my $lSize;
+
+ # Is this an IO object or a file expression?
+ my $oFileIo = defined($xFileExp) ? (ref($xFileExp) ? $xFileExp : $self->openRead($self->pathGet($xFileExp))) : undef;
+
+ if (defined($oFileIo))
+ {
+ $lSize = 0;
+ my $oShaIo = new pgBackRest::Storage::Filter::Sha($oFileIo);
+ my $lSizeRead;
+
+ do
+ {
+ my $tContent;
+ $lSizeRead = $oShaIo->read(\$tContent, $self->{lBufferMax});
+ $lSize += $lSizeRead;
+ }
+ while ($lSizeRead != 0);
+
+ # Close the file
+ $oShaIo->close();
+
+ # Get the hash
+ $strHash = $oShaIo->result(STORAGE_FILTER_SHA);
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'strHash', value => $strHash},
+ {name => 'lSize', value => $lSize}
+ );
+}
+
+####################################################################################################################################
+# info - get information for path/file
+####################################################################################################################################
+sub info
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathFileExp,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::fileStat', \@_,
+ {name => 'strPathFileExp'},
+ {name => 'bIgnoreMissing', default => false},
+ );
+
+ # Stat the path/file
+ my $oInfo = $self->driver()->info($self->pathGet($strPathFileExp), $bIgnoreMissing);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oInfo', value => $oInfo, trace => true}
+ );
+}
+
+####################################################################################################################################
+# linkCreate - create a link
+####################################################################################################################################
+sub linkCreate
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strSourcePathFileExp,
+ $strDestinationLinkExp,
+ $bHard,
+ $bRelative,
+ $bPathCreate,
+ $bIgnoreExists,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->linkCreate', \@_,
+ {name => 'strSourcePathFileExp'},
+ {name => 'strDestinationLinkExp'},
+ {name => 'bHard', optional=> true, default => false},
+ {name => 'bRelative', optional=> true, default => false},
+ {name => 'bPathCreate', optional=> true, default => true},
+ {name => 'bIgnoreExists', optional => true, default => false},
+ );
+
+ # Get source and destination paths
+ my $strSourcePathFile = $self->pathGet($strSourcePathFileExp);
+ my $strDestinationLink = $self->pathGet($strDestinationLinkExp);
+
+ # Generate relative path if requested
+ if ($bRelative)
+ {
+ # Determine how much of the paths are common
+ my @strySource = split('/', $strSourcePathFile);
+ my @stryDestination = split('/', $strDestinationLink);
+
+ while (defined($strySource[0]) && defined($stryDestination[0]) && $strySource[0] eq $stryDestination[0])
+ {
+ shift(@strySource);
+ shift(@stryDestination);
+ }
+
+ # Add relative path sections
+ $strSourcePathFile = '';
+
+ for (my $iIndex = 0; $iIndex < @stryDestination - 1; $iIndex++)
+ {
+ $strSourcePathFile .= '../';
+ }
+
+ # Add path to source
+ $strSourcePathFile .= join('/', @strySource);
+
+ logDebugMisc
+ (
+ $strOperation, 'apply relative path',
+ {name => 'strSourcePathFile', value => $strSourcePathFile, trace => true}
+ );
+ }
+
+ # Create the link
+ $self->driver()->linkCreate(
+ $strSourcePathFile, $strDestinationLink, {bHard => $bHard, bPathCreate => $bPathCreate, bIgnoreExists => $bIgnoreExists});
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# list - list all files/paths in path
+####################################################################################################################################
+sub list
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ $strExpression,
+ $strSortOrder,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->list', \@_,
+ {name => 'strPathExp'},
+ {name => 'strExpression', optional => true},
+ {name => 'strSortOrder', optional => true, default => 'forward'},
+ {name => 'bIgnoreMissing', optional => true, default => false},
+ );
+
+ # Get file list
+ my $rstryFileList = $self->driver()->list($self->pathGet($strPathExp), {bIgnoreMissing => $bIgnoreMissing});
+
+ # Apply expression if defined
+ if (defined($strExpression))
+ {
+ @{$rstryFileList} = grep(/$strExpression/i, @{$rstryFileList});
+ }
+
+ # Reverse sort
+ if ($strSortOrder eq 'reverse')
+ {
+ @{$rstryFileList} = sort {$b cmp $a} @{$rstryFileList};
+ }
+ # Normal sort
+ else
+ {
+ @{$rstryFileList} = sort @{$rstryFileList};
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'stryFileList', value => $rstryFileList}
+ );
+}
+
+####################################################################################################################################
+# manifest - build path/file/link manifest starting with base path and including all subpaths
+####################################################################################################################################
+sub manifest
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->manifest', \@_,
+ {name => 'strPathExp'},
+ );
+
+ my $hManifest = $self->driver()->manifest($self->pathGet($strPathExp));
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'hManifest', value => $hManifest, trace => true}
+ );
+}
+
+####################################################################################################################################
+# move - move path/file
+####################################################################################################################################
+sub move
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strSourcePathFileExp,
+ $strDestinationPathFileExp,
+ $bPathCreate,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->move', \@_,
+ {name => 'strSourcePathExp'},
+ {name => 'strDestinationPathExp'},
+ {name => 'bPathCreate', optional => true, default => false, trace => true},
+ );
+
+ # Set operation variables
+ $self->driver()->move(
+ $self->pathGet($strSourcePathFileExp), $self->pathGet($strDestinationPathFileExp), {bPathCreate => $bPathCreate});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation
+ );
+}
+
+####################################################################################################################################
+# openRead - open file for reading
+####################################################################################################################################
+sub openRead
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xFileExp,
+ $bIgnoreMissing,
+ $rhyFilter,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->openRead', \@_,
+ {name => 'xFileExp'},
+ {name => 'bIgnoreMissing', optional => true, default => false},
+ {name => 'rhyFilter', optional => true},
+ );
+
+ # Need to push this down to drivers so errors do not appear in the log
+ my $oFileIo = $self->driver()->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing});
+
+ # Apply filters if file is defined
+ if (defined($rhyFilter) && defined($oFileIo))
+ {
+ foreach my $rhFilter (@{$rhyFilter})
+ {
+ $oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}});
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oFileIo', value => $oFileIo, trace => true},
+ );
+}
+
+####################################################################################################################################
+# openWrite - open file for writing
+####################################################################################################################################
+sub openWrite
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xFileExp,
+ $strMode,
+ $strUser,
+ $strGroup,
+ $lTimestamp,
+ $bAtomic,
+ $bPathCreate,
+ $rhyFilter,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->openWrite', \@_,
+ {name => 'xFileExp'},
+ {name => 'strMode', optional => true, default => $self->{strDefaultFileMode}},
+ {name => 'strUser', optional => true},
+ {name => 'strGroup', optional => true},
+ {name => 'lTimestamp', optional => true},
+ {name => 'bAtomic', optional => true, default => false},
+ {name => 'bPathCreate', optional => true, default => false},
+ {name => 'rhyFilter', optional => true},
+ );
+
+ # Open the file
+ my $oFileIo = $self->driver()->openWrite($self->pathGet($xFileExp),
+ {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate,
+ bAtomic => $bAtomic});
+
+ # Apply filters if file is defined
+ if (defined($rhyFilter))
+ {
+ foreach my $rhFilter (reverse(@{$rhyFilter}))
+ {
+ $oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}});
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oFileIo', value => $oFileIo, trace => true},
+ );
+}
+
+####################################################################################################################################
+# owner - change ownership of path/file
+####################################################################################################################################
+sub owner
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathFileExp,
+ $strUser,
+ $strGroup
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->owner', \@_,
+ {name => 'strPathFileExp'},
+ {name => 'strUser', required => false},
+ {name => 'strGroup', required => false}
+ );
+
+ # Set ownership
+ $self->driver()->owner($self->pathGet($strPathFileExp), {strUser => $strUser, strGroup => $strGroup});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation
+ );
+}
+
+####################################################################################################################################
+# pathCreate - create path
+####################################################################################################################################
+sub pathCreate
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ $strMode,
+ $bIgnoreExists,
+ $bCreateParent,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathCreate', \@_,
+ {name => 'strPathExp'},
+ {name => 'strMode', optional => true, default => $self->{strDefaultPathMode}},
+ {name => 'bIgnoreExists', optional => true, default => false},
+ {name => 'bCreateParent', optional => true, default => false},
+ );
+
+ # Create path
+ $self->driver()->pathCreate(
+ $self->pathGet($strPathExp), {strMode => $strMode, bIgnoreExists => $bIgnoreExists, bCreateParent => $bCreateParent});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation
+ );
+}
+
+####################################################################################################################################
+# pathExists - check if path exists
+####################################################################################################################################
+sub pathExists
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathExists', \@_,
+ {name => 'strPathExp'},
+ );
+
+ # Check exists
+ my $bExists = $self->driver()->pathExists($self->pathGet($strPathExp));
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bExists', value => $bExists}
+ );
+}
+
+####################################################################################################################################
+# pathGet - resolve a path expression into an absolute path
+####################################################################################################################################
+sub pathGet
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp, # File that that needs to be translated to a path
+ $bTemp, # Return the temp file name
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathGet', \@_,
+ {name => 'strPathExp', required => false, trace => true},
+ {name => 'bTemp', optional => true, default => false, trace => true},
+ );
+
+ # Path and file to be returned
+ my $strPath;
+ my $strFile;
+
+ # Is this an absolute path type?
+ my $bAbsolute = false;
+
+ if (defined($strPathExp) && index($strPathExp, qw(/)) == 0)
+ {
+ $bAbsolute = true;
+ $strPath = $strPathExp;
+ }
+ else
+ {
+ # Is it a rule type
+ if (defined($strPathExp) && index($strPathExp, qw(<)) == 0)
+ {
+ # Extract the rule type
+ my $iPos = index($strPathExp, qw(>));
+
+ if ($iPos == -1)
+ {
+ confess &log(ASSERT, "found < but not > in '${strPathExp}'");
+ }
+
+ my $strType = substr($strPathExp, 0, $iPos + 1);
+
+ # Extract the filename
+ if ($iPos < length($strPathExp) - 1)
+ {
+ $strFile = substr($strPathExp, $iPos + 2);
+ }
+
+ # Lookup the rule
+ if (!defined($self->{hRule}->{$strType}))
+ {
+ confess &log(ASSERT, "storage rule '${strType}' does not exist");
+ }
+
+ # If rule is a ref then call the function
+ if (ref($self->{hRule}->{$strType}))
+ {
+ $strPath = $self->pathBase();
+ $strFile = $self->{hRule}{$strType}{fnRule}->($strType, $strFile, $self->{hRule}{$strType}{xData});
+ }
+ # Else get the path
+ else
+ {
+ $strPath = $self->pathBase() . ($self->pathBase() =~ /\/$/ ? '' : qw{/}) . $self->{hRule}->{$strType};
+ }
+ }
+ # Else it must be relative
+ else
+ {
+ $strPath = $self->pathBase();
+ $strFile = $strPathExp;
+ }
+ }
+
+ # Make sure a temp file is valid for this type and file
+ if ($bTemp)
+ {
+ # Error when temp files are not allowed
+ if (!$self->{bAllowTemp})
+ {
+ confess &log(ASSERT, "temp file not supported for storage '" . $self->pathBase() . "'");
+ }
+
+ # The file must be defined
+ if (!$bAbsolute)
+ {
+ if (!defined($strFile))
+ {
+ confess &log(ASSERT, 'file part must be defined when temp file specified');
+ }
+ }
+ }
+
+ # Combine path and file
+ $strPath .= defined($strFile) ? ($strPath =~ /\/$/ ? '' : qw{/}) . "${strFile}" : '';
+
+ # Add temp extension
+ $strPath .= $bTemp ? ".$self->{strTempExtension}" : '';
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'strPath', value => $strPath, trace => true}
+ );
+}
+
+####################################################################################################################################
+# pathSync - perform driver sync operation on path
+####################################################################################################################################
+sub pathSync
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathExp,
+ $bRecurse,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathSync', \@_,
+ {name => 'strPathExp'},
+ {name => 'bRecurse', default => false},
+ );
+
+ $self->driver()->pathSync($self->pathGet($strPathExp), {bRecurse => true});
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# remove - remove path/file
+####################################################################################################################################
+sub remove
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xstryPathFileExp,
+ $bIgnoreMissing,
+ $bRecurse,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->remove', \@_,
+ {name => 'strPathFileExp'},
+ {name => 'bIgnoreMissing', optional => true, default => true},
+ {name => 'bRecurse', optional => true, default => false, trace => true},
+ );
+
+ # Evaluate expressions for all files
+ my @stryPathFileExp;
+
+ if (ref($xstryPathFileExp))
+ {
+ foreach my $strPathFileExp (@{$xstryPathFileExp})
+ {
+ push(@stryPathFileExp, $self->pathGet($strPathFileExp));
+ }
+ }
+
+ # Remove path(s)/file(s)
+ my $bRemoved = $self->driver()->remove(
+ ref($xstryPathFileExp) ? \@stryPathFileExp : $self->pathGet($xstryPathFileExp),
+ {bIgnoreMissing => $bIgnoreMissing, bRecurse => $bRecurse});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bRemoved', value => $bRemoved}
+ );
+}
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub pathBase {shift->{strPathBase}}
+sub driver {shift->{oDriver}}
+
+1;
diff --git a/lib/pgBackRest/Storage/Posix/Driver.pm b/lib/pgBackRest/Storage/Posix/Driver.pm
new file mode 100644
index 000000000..26bc398bc
--- /dev/null
+++ b/lib/pgBackRest/Storage/Posix/Driver.pm
@@ -0,0 +1,1000 @@
+####################################################################################################################################
+# Posix Storage Driver
+#
+# Implements storage functions for Posix-compliant file systems.
+####################################################################################################################################
+package pgBackRest::Storage::Posix::Driver;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+ our @EXPORT = qw();
+use File::Basename qw(basename dirname);
+use Fcntl qw(:mode);
+use File::stat qw{lstat};
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::Posix::FileRead;
+use pgBackRest::Storage::Posix::FileWrite;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant STORAGE_POSIX_DRIVER => __PACKAGE__;
+ push @EXPORT, qw(STORAGE_POSIX_DRIVER);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Create the class hash
+ my $self = {};
+ bless $self, $class;
+
+ # Assign function parameters, defaults, and log debug info
+ (
+ my $strOperation,
+ $self->{bFileSync},
+ $self->{bPathSync},
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'bFileSync', optional => true, default => true, trace => true},
+ {name => 'bPathSync', optional => true, default => true, trace => true},
+ );
+
+ # Set default temp extension
+ $self->{strTempExtension} = 'tmp';
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self, trace => true}
+ );
+}
+
+####################################################################################################################################
+# exists - check if a path or file exists
+####################################################################################################################################
+sub exists
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFile,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->exists', \@_,
+ {name => 'strFile', trace => true},
+ );
+
+ # Does the path/file exist?
+ my $bExists = true;
+ my $oStat = lstat($strFile);
+
+ # Use stat to test if file exists
+ if (defined($oStat))
+ {
+ # Check that it is actually a file
+ $bExists = !S_ISDIR($oStat->mode) ? true : false;
+ }
+ else
+ {
+ # If the error is not entry missing, then throw error
+ if (!$OS_ERROR{ENOENT})
+ {
+ logErrorResult(ERROR_FILE_EXISTS, "unable to test if file '${strFile}' exists", $OS_ERROR);
+ }
+
+ $bExists = false;
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bExists', value => $bExists, trace => true}
+ );
+}
+
+####################################################################################################################################
+# info - get information for path/file
+####################################################################################################################################
+sub info
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPathFile,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->info', \@_,
+ {name => 'strFile', trace => true},
+ {name => 'bIgnoreMissing', default => false, trace => true},
+ );
+
+ # Stat the path/file
+ my $oInfo = lstat($strPathFile);
+
+ # Check for errors
+ if (!defined($oInfo))
+ {
+ if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
+ {
+ logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to stat '${strPathFile}'", $OS_ERROR);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oInfo', value => $oInfo, trace => true}
+ );
+}
+
+####################################################################################################################################
+# linkCreate
+####################################################################################################################################
+sub linkCreate
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strSourcePathFile,
+ $strDestinationLink,
+ $bHard,
+ $bPathCreate,
+ $bIgnoreExists,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->linkCreate', \@_,
+ {name => 'strSourcePathFile', trace => true},
+ {name => 'strDestinationLink', trace => true},
+ {name => 'bHard', optional=> true, default => false, trace => true},
+ {name => 'bPathCreate', optional=> true, default => true, trace => true},
+ {name => 'bIgnoreExists', optional => true, default => false, trace => true},
+ );
+
+ if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink)))
+ {
+ my $strMessage = "unable to create link '${strDestinationLink}'";
+
+ # If parent path or source is missing
+ if ($OS_ERROR{ENOENT})
+ {
+ # Check if source is missing
+ if (!$self->exists($strSourcePathFile))
+ {
+ confess &log(ERROR, "${strMessage} because source '${strSourcePathFile}' does not exist", ERROR_FILE_MISSING);
+ }
+
+ if (!$bPathCreate)
+ {
+ confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING);
+ }
+
+ # Create parent path
+ $self->pathCreate(dirname($strDestinationLink), {bIgnoreExists => true, bCreateParent => true});
+
+ # Create link
+ $self->linkCreate($strSourcePathFile, $strDestinationLink, {bHard => $bHard});
+ }
+ # Else if link already exists
+ elsif ($OS_ERROR{EEXIST})
+ {
+ if (!$bIgnoreExists)
+ {
+ confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS);
+ }
+ }
+ else
+ {
+ logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# linkDestination - get destination of symlink
+####################################################################################################################################
+sub linkDestination
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strLink,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->linkDestination', \@_,
+ {name => 'strLink', trace => true},
+ );
+
+ # Get link destination
+ my $strLinkDestination = readlink($strLink);
+
+ # Check for errors
+ if (!defined($strLinkDestination))
+ {
+ logErrorResult(
+ $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to get destination for link ${strLink}", $OS_ERROR);
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'strLinkDestination', value => $strLinkDestination, trace => true}
+ );
+}
+
+####################################################################################################################################
+# list - list all files/paths in path
+####################################################################################################################################
+sub list
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->list', \@_,
+ {name => 'strPath', trace => true},
+ {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+ );
+
+ # Working variables
+ my @stryFileList;
+ my $hPath;
+
+ # Attempt to open the path
+ if (opendir($hPath, $strPath))
+ {
+ @stryFileList = grep(!/^(\.)|(\.\.)$/i, readdir($hPath));
+ close($hPath);
+ }
+ # Else process errors
+ else
+ {
+ # Ignore the error if the file is missing and missing files should be ignored
+ if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
+ {
+ logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to read path '${strPath}'", $OS_ERROR);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'stryFileList', value => \@stryFileList, ref => true, trace => true}
+ );
+}
+
+####################################################################################################################################
+# manifest - build path/file/link manifest starting with base path and including all subpaths
+####################################################################################################################################
+sub manifest
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->manifest', \@_,
+ {name => 'strPath', trace => true},
+ );
+
+ # Generate the manifest
+ my $hManifest = {};
+ $self->manifestRecurse($strPath, undef, 0, $hManifest);
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'hManifest', value => $hManifest, trace => true}
+ );
+}
+
+sub manifestRecurse
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ $strSubPath,
+ $iDepth,
+ $hManifest,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '::manifestRecurse', \@_,
+ {name => 'strPath', trace => true},
+ {name => 'strSubPath', required => false, trace => true},
+ {name => 'iDepth', default => 0, trace => true},
+ {name => 'hManifest', required => false, trace => true},
+ );
+
+ # Set operation and debug strings
+ my $strPathRead = $strPath . (defined($strSubPath) ? "/${strSubPath}" : '');
+ my $hPath;
+ my $strFilter;
+
+ # If this is the top level stat the path to discover if it is actually a file
+ if ($iDepth == 0 && !S_ISDIR(($self->info($strPathRead))->mode))
+ {
+ $strFilter = basename($strPathRead);
+ $strPathRead = dirname($strPathRead);
+ }
+
+ # Get a list of all files in the path (including .)
+ my @stryFileList = @{$self->list($strPathRead, {bIgnoreMissing => $iDepth != 0})};
+ unshift(@stryFileList, '.');
+ my $hFileStat = $self->manifestList($strPathRead, \@stryFileList);
+
+ # Loop through all subpaths/files in the path
+ foreach my $strFile (keys(%{$hFileStat}))
+ {
+ # Skip this file if it does not match the filter
+ if (defined($strFilter) && $strFile ne $strFilter)
+ {
+ next;
+ }
+
+ my $strManifestFile = $iDepth == 0 ? $strFile : ($strSubPath . ($strFile eq qw(.) ? '' : "/${strFile}"));
+ $hManifest->{$strManifestFile} = $hFileStat->{$strFile};
+
+ # Recurse into directories
+ if ($hManifest->{$strManifestFile}{type} eq 'd' && $strFile ne qw(.))
+ {
+ $self->manifestRecurse($strPath, $strManifestFile, $iDepth + 1, $hManifest);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+sub manifestList
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ $stryFile,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->manifestList', \@_,
+ {name => 'strPath', trace => true},
+ {name => 'stryFile', trace => true},
+ );
+
+ my $hFileStat = {};
+
+ foreach my $strFile (@{$stryFile})
+ {
+ $hFileStat->{$strFile} = $self->manifestStat("${strPath}" . ($strFile eq qw(.) ? '' : "/${strFile}"));
+
+ if (!defined($hFileStat->{$strFile}))
+ {
+ delete($hFileStat->{$strFile});
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'hFileStat', value => $hFileStat, trace => true}
+ );
+}
+
+sub manifestStat
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFile,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->manifestStat', \@_,
+ {name => 'strFile', trace => true},
+ );
+
+ # Stat the path/file, ignoring any that are missing
+ my $oStat = $self->info($strFile, {bIgnoreMissing => true});
+
+ # Generate file data if stat succeeded (i.e. file exists)
+ my $hFile;
+
+ if (defined($oStat))
+ {
+ # Check for regular file
+ if (S_ISREG($oStat->mode))
+ {
+ $hFile->{type} = 'f';
+
+ # Get size
+ $hFile->{size} = $oStat->size;
+
+ # Get modification time
+ $hFile->{modification_time} = $oStat->mtime;
+ }
+ # Check for directory
+ elsif (S_ISDIR($oStat->mode))
+ {
+ $hFile->{type} = 'd';
+ }
+ # Check for link
+ elsif (S_ISLNK($oStat->mode))
+ {
+ $hFile->{type} = 'l';
+ $hFile->{link_destination} = $self->linkDestination($strFile);
+ }
+ # Not a recognized type
+ else
+ {
+ confess &log(ERROR, "${strFile} is not of type directory, file, or link", ERROR_FILE_INVALID);
+ }
+
+ # Get user name
+ $hFile->{user} = getpwuid($oStat->uid);
+
+ # Get group name
+ $hFile->{group} = getgrgid($oStat->gid);
+
+ # Get mode
+ if ($hFile->{type} ne 'l')
+ {
+ $hFile->{mode} = sprintf('%04o', S_IMODE($oStat->mode));
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'hFile', value => $hFile, trace => true}
+ );
+}
+
+####################################################################################################################################
+# move - move path/file
+####################################################################################################################################
+sub move
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strSourceFile,
+ $strDestinationFile,
+ $bPathCreate,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->move', \@_,
+ {name => 'strSourceFile', trace => true},
+ {name => 'strDestinationFile', trace => true},
+ {name => 'bPathCreate', default => false, trace => true},
+ );
+
+ # Get source and destination paths
+ my $strSourcePathFile = dirname($strSourceFile);
+ my $strDestinationPathFile = dirname($strDestinationFile);
+
+ # Move the file
+ if (!rename($strSourceFile, $strDestinationFile))
+ {
+ my $strMessage = "unable to move '${strSourceFile}'";
+
+ # If something is missing determine if it is the source or destination
+ if ($OS_ERROR{ENOENT})
+ {
+ if (!$self->exists($strSourceFile))
+ {
+ logErrorResult(ERROR_FILE_MISSING, "${strMessage} because it is missing");
+ }
+
+ if ($bPathCreate)
+ {
+ # Attempt to create the path - ignore exists here in case another process creates it first
+ $self->pathCreate($strDestinationPathFile, {bCreateParent => true, bIgnoreExists => true});
+
+ # Try move again
+ $self->move($strSourceFile, $strDestinationFile);
+ }
+ else
+ {
+ logErrorResult(ERROR_PATH_MISSING, "${strMessage} to missing path '${strDestinationPathFile}'");
+ }
+ }
+ # Else raise the error
+ else
+ {
+ logErrorResult(ERROR_FILE_MOVE, "${strMessage} to '${strDestinationFile}'", $OS_ERROR);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# openRead - open file for reading
+####################################################################################################################################
+sub openRead
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFile,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->openRead', \@_,
+ {name => 'strFile', trace => true},
+ {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+ );
+
+ my $oFileIO = new pgBackRest::Storage::Posix::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oFileIO', value => $oFileIO, trace => true},
+ );
+}
+
+####################################################################################################################################
+# openWrite - open file for writing
+####################################################################################################################################
+sub openWrite
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFile,
+ $strMode,
+ $strUser,
+ $strGroup,
+ $lTimestamp,
+ $bPathCreate,
+ $bAtomic,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->openWrite', \@_,
+ {name => 'strFile', trace => true},
+ {name => 'strMode', optional => true, trace => true},
+ {name => 'strUser', optional => true, trace => true},
+ {name => 'strGroup', optional => true, trace => true},
+ {name => 'lTimestamp', optional => true, trace => true},
+ {name => 'bPathCreate', optional => true, trace => true},
+ {name => 'bAtomic', optional => true, trace => true},
+ );
+
+ my $oFileIO = new pgBackRest::Storage::Posix::FileWrite(
+ $self, $strFile,
+ {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate,
+ bAtomic => $bAtomic, bSync => $self->{bFileSync}});
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'oFileIO', value => $oFileIO, trace => true},
+ );
+}
+
+####################################################################################################################################
+# owner - change ownership of path/file
+####################################################################################################################################
+sub owner
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strFilePath,
+ $strUser,
+ $strGroup,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->owner', \@_,
+ {name => 'strFilePath', trace => true},
+ {name => 'strUser', optional => true, trace => true},
+ {name => 'strGroup', optional => true, trace => true},
+ );
+
+ # Only proceed if user or group was specified
+ if (defined($strUser) || defined($strGroup))
+ {
+ my $strMessage = "unable to set ownership for '${strFilePath}'";
+ my $iUserId;
+ my $iGroupId;
+
+ # If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that
+ # both user and group be set.
+ if (!(defined($strUser) && defined($strGroup)))
+ {
+ my $oStat = $self->info($strFilePath);
+
+ if (!defined($strUser))
+ {
+ $iUserId = $oStat->uid;
+ }
+
+ if (!defined($strGroup))
+ {
+ $iGroupId = $oStat->gid;
+ }
+ }
+
+ # Lookup user if specified
+ if (defined($strUser))
+ {
+ $iUserId = getpwnam($strUser);
+
+ if (!defined($iUserId))
+ {
+ logErrorResult(ERROR_FILE_OWNER, "${strMessage} because user '${strUser}' does not exist");
+ }
+ }
+
+ # Lookup group if specified
+ if (defined($strGroup))
+ {
+ $iGroupId = getgrnam($strGroup);
+
+ if (!defined($iGroupId))
+ {
+ logErrorResult(ERROR_FILE_OWNER, "${strMessage} because group '${strGroup}' does not exist");
+ }
+ }
+
+ # Set ownership on the file
+ if (!chown($iUserId, $iGroupId, $strFilePath))
+ {
+ if ($OS_ERROR{ENOENT})
+ {
+ logErrorResult(ERROR_FILE_OWNER, "${strMessage} because it is missing");
+ }
+
+ logErrorResult(ERROR_FILE_OWNER, "${strMessage}", $OS_ERROR);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# pathCreate - create path
+####################################################################################################################################
+sub pathCreate
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ $strMode,
+ $bIgnoreExists,
+ $bCreateParent,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathCreate', \@_,
+ {name => 'strPath', trace => true},
+ {name => 'strMode', optional => true, default => '0750', trace => true},
+ {name => 'bIgnoreExists', optional => true, default => false, trace => true},
+ {name => 'bCreateParent', optional => true, default => false, trace => true},
+ );
+
+ # Attempt to create the directory
+ if (!mkdir($strPath, oct($strMode)))
+ {
+ my $strMessage = "unable to create path '${strPath}'";
+
+ # If parent path is missing
+ if ($OS_ERROR{ENOENT})
+ {
+ if (!$bCreateParent)
+ {
+ confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING);
+ }
+
+ # Create parent path
+ $self->pathCreate(dirname($strPath), {strMode => $strMode, bIgnoreExists => true, bCreateParent => $bCreateParent});
+
+ # Create path
+ $self->pathCreate($strPath, {strMode => $strMode, bIgnoreExists => true});
+ }
+ # Else if path already exists
+ elsif ($OS_ERROR{EEXIST})
+ {
+ if (!$bIgnoreExists)
+ {
+ confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS);
+ }
+ }
+ else
+ {
+ logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR);
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# pathExists - check if path exists
+####################################################################################################################################
+sub pathExists
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathExists', \@_,
+ {name => 'strPath', trace => true},
+ );
+
+ # Does the path/file exist?
+ my $bExists = true;
+ my $oStat = lstat($strPath);
+
+ # Use stat to test if path exists
+ if (defined($oStat))
+ {
+ # Check that it is actually a path
+ $bExists = S_ISDIR($oStat->mode) ? true : false;
+ }
+ else
+ {
+ # If the error is not entry missing, then throw error
+ if (!$OS_ERROR{ENOENT})
+ {
+ logErrorResult(ERROR_FILE_EXISTS, "unable to test if path '${strPath}' exists", $OS_ERROR);
+ }
+
+ $bExists = false;
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bExists', value => $bExists, trace => true}
+ );
+}
+
+####################################################################################################################################
+# pathSync - perform fsync on path
+####################################################################################################################################
+sub pathSync
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $strPath,
+ $bRecurse,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->pathSync', \@_,
+ {name => 'strPath', trace => true},
+ {name => 'bRecurse', default => false, trace => true},
+ );
+
+ if ($bRecurse)
+ {
+ my $oManifest = $self->manifest($strPath);
+
+ # Iterate all files in the manifest
+ foreach my $strFile (sort(keys(%{$oManifest})))
+ {
+ # Only sync if this is a directory
+ if ($oManifest->{$strFile}{type} eq 'd')
+ {
+ # If current directory
+ if ($strFile eq '.')
+ {
+ $self->pathSync($strPath);
+ }
+ # Else a subdirectory
+ else
+ {
+ $self->pathSync("${strPath}/${strFile}");
+ }
+ }
+ }
+ }
+ else
+ {
+ open(my $hPath, "<", $strPath)
+ or confess &log(ERROR, "unable to open '${strPath}' for sync", ERROR_PATH_OPEN);
+ open(my $hPathDup, ">&", $hPath)
+ or confess &log(ERROR, "unable to duplicate '${strPath}' handle for sync", ERROR_PATH_OPEN);
+
+ $hPathDup->sync()
+ or confess &log(ERROR, "unable to sync path '${strPath}'", ERROR_PATH_SYNC);
+
+ close($hPathDup);
+ close($hPath);
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# remove - remove path/file
+####################################################################################################################################
+sub remove
+{
+ my $self = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $xstryPathFile,
+ $bIgnoreMissing,
+ $bRecurse,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->remove', \@_,
+ {name => 'xstryPathFile', trace => true},
+ {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+ {name => 'bRecurse', optional => true, default => false, trace => true},
+ );
+
+ # Working variables
+ my $bRemoved = true;
+
+ # Remove a tree
+ if ($bRecurse)
+ {
+ my $oManifest = $self->manifest($xstryPathFile);
+
+ # Iterate all files in the manifest
+ foreach my $strFile (sort({$b cmp $a} keys(%{$oManifest})))
+ {
+ # remove directory
+ if ($oManifest->{$strFile}{type} eq 'd')
+ {
+ my $xstryPathFileRemove = $strFile eq '.' ? $xstryPathFile : "${xstryPathFile}/${strFile}";
+
+ if (!rmdir($xstryPathFileRemove))
+ {
+ # If any error but missing then raise the error
+ if (!$OS_ERROR{ENOENT})
+ {
+ logErrorResult(ERROR_PATH_REMOVE, "unable to remove path '${strFile}'", $OS_ERROR);
+ }
+ }
+ }
+ # Remove file
+ else
+ {
+ $self->remove("${xstryPathFile}/${strFile}", {bIgnoreMissing => true});
+ }
+ }
+ }
+ # Only remove the specified file
+ else
+ {
+ foreach my $strFile (ref($xstryPathFile) ? @{$xstryPathFile} : ($xstryPathFile))
+ {
+ if (unlink($strFile) != 1)
+ {
+ $bRemoved = false;
+
+ # If path exists then throw the error
+ if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
+ {
+ logErrorResult(
+ $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to remove file '${strFile}'", $OS_ERROR);
+ }
+ }
+ }
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'bRemoved', value => $bRemoved, trace => true}
+ );
+}
+
+####################################################################################################################################
+# Getters/Setters
+####################################################################################################################################
+sub capability {true}
+sub className {STORAGE_POSIX_DRIVER}
+sub tempExtension {shift->{strTempExtension}}
+sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift}
+
+1;
diff --git a/lib/pgBackRest/Storage/Posix/FileRead.pm b/lib/pgBackRest/Storage/Posix/FileRead.pm
new file mode 100644
index 000000000..5dce51b73
--- /dev/null
+++ b/lib/pgBackRest/Storage/Posix/FileRead.pm
@@ -0,0 +1,104 @@
+####################################################################################################################################
+# Posix File Read
+####################################################################################################################################
+package pgBackRest::Storage::Posix::FileRead;
+use parent 'pgBackRest::Common::Io::Handle';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Fcntl qw(O_RDONLY);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $oDriver,
+ $strName,
+ $bIgnoreMissing,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'oDriver', trace => true},
+ {name => 'strName', trace => true},
+ {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+ );
+
+ # Open the file
+ my $fhFile;
+
+ if (!sysopen($fhFile, $strName, O_RDONLY))
+ {
+ if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
+ {
+ logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to open '${strName}'", $OS_ERROR);
+ }
+
+ undef($fhFile);
+ }
+
+ # Create IO object if open succeeded
+ my $self;
+
+ if (defined($fhFile))
+ {
+ # Set file mode to binary
+ binmode($fhFile);
+
+ # Create the class hash
+ $self = $class->SUPER::new("'${strName}'", $fhFile);
+ bless $self, $class;
+
+ # Set variables
+ $self->{oDriver} = $oDriver;
+ $self->{strName} = $strName;
+ $self->{fhFile} = $fhFile;
+ }
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self, trace => true}
+ );
+}
+
+####################################################################################################################################
+# close - close the file
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ if (defined($self->handle()))
+ {
+ # Close the file
+ close($self->handle());
+ undef($self->{fhFile});
+
+ # Close parent
+ $self->SUPER::close();
+ }
+
+ return true;
+}
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub handle {shift->{fhFile}}
+
+1;
diff --git a/lib/pgBackRest/Storage/Posix/FileWrite.pm b/lib/pgBackRest/Storage/Posix/FileWrite.pm
new file mode 100644
index 000000000..6e3e95496
--- /dev/null
+++ b/lib/pgBackRest/Storage/Posix/FileWrite.pm
@@ -0,0 +1,192 @@
+####################################################################################################################################
+# Posix File Write
+####################################################################################################################################
+package pgBackRest::Storage::Posix::FileWrite;
+use parent 'pgBackRest::Common::Io::Handle';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
+use File::Basename qw(dirname);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+use pgBackRest::Common::Io::Handle;
+use pgBackRest::Storage::Base;
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+sub new
+{
+ my $class = shift;
+
+ # Assign function parameters, defaults, and log debug info
+ my
+ (
+ $strOperation,
+ $oDriver,
+ $strName,
+ $strMode,
+ $strUser,
+ $strGroup,
+ $lTimestamp,
+ $bPathCreate,
+ $bAtomic,
+ $bSync,
+ ) =
+ logDebugParam
+ (
+ __PACKAGE__ . '->new', \@_,
+ {name => 'oDriver', trace => true},
+ {name => 'strName', trace => true},
+ {name => 'strMode', optional => true, trace => true},
+ {name => 'strUser', optional => true, trace => true},
+ {name => 'strGroup', optional => true, trace => true},
+ {name => 'lTimestamp', optional => true, trace => true},
+ {name => 'bPathCreate', optional => true, default => false, trace => true},
+ {name => 'bAtomic', optional => true, default => false, trace => true},
+ {name => 'bSync', optional => true, default => true, trace => true},
+ );
+
+ # Create the class hash
+ my $self = $class->SUPER::new("'${strName}'");
+ bless $self, $class;
+
+ # Set variables
+ $self->{oDriver} = $oDriver;
+ $self->{strName} = $strName;
+ $self->{strMode} = $strMode;
+ $self->{strUser} = $strUser;
+ $self->{strGroup} = $strGroup;
+ $self->{lTimestamp} = $lTimestamp;
+ $self->{bPathCreate} = $bPathCreate;
+ $self->{bAtomic} = $bAtomic;
+ $self->{bSync} = $bSync;
+
+ # If atomic create temp filename
+ if ($self->{bAtomic})
+ {
+ # Create temp file name
+ $self->{strNameTmp} = "$self->{strName}." . $self->{oDriver}->tempExtension();
+ }
+
+ # Open file on first write to avoid creating extraneous files on error
+ $self->{bOpened} = false;
+
+ # Return from function and log return values if any
+ return logDebugReturn
+ (
+ $strOperation,
+ {name => 'self', value => $self, trace => true}
+ );
+}
+
+####################################################################################################################################
+# open - open the file
+####################################################################################################################################
+sub open
+{
+ my $self = shift;
+
+ # Get the file name
+ my $strFile = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName};
+
+ # Open the file
+ if (!sysopen(
+ $self->{fhFile}, $strFile, O_WRONLY | O_CREAT | O_TRUNC, oct(defined($self->{strMode}) ? $self->{strMode} : '0666')))
+ {
+ # If the path does not exist create it if requested
+ if ($OS_ERROR{ENOENT} && $self->{bPathCreate})
+ {
+ $self->{oDriver}->pathCreate(dirname($strFile), {bIgnoreExists => true, bCreateParent => true});
+ $self->{bPathCreate} = false;
+ return $self->open();
+ }
+ }
+
+ # Set file mode to binary
+ binmode($self->{fhFile});
+
+ # Set the owner
+ $self->{oDriver}->owner($strFile, {strUser => $self->{strUser}, strGroup => $self->{strGroup}});
+
+ # Set handle
+ $self->handleWriteSet($self->{fhFile});
+
+ # Mark file as opened
+ $self->{bOpened} = true;
+
+ return true;
+}
+
+####################################################################################################################################
+# write - write data to a file
+####################################################################################################################################
+sub write
+{
+ my $self = shift;
+ my $rtBuffer = shift;
+
+ # Open file if it is not open already
+ $self->open() if !$self->opened();
+
+ return $self->SUPER::write($rtBuffer);
+}
+
+####################################################################################################################################
+# close - close the file
+####################################################################################################################################
+sub close
+{
+ my $self = shift;
+
+ if (defined($self->handle()))
+ {
+ # Sync the file
+ if ($self->{bSync})
+ {
+ $self->handle()->sync();
+ }
+
+ # Close the file
+ close($self->handle());
+ undef($self->{fhFile});
+
+ # Get current filename
+ my $strCurrentName = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName};
+
+ # Set the modification time
+ if (defined($self->{lTimestamp}))
+ {
+ utime(time(), $self->{lTimestamp}, $strCurrentName)
+ or logErrorResult(ERROR_FILE_WRITE, "unable to set time for '${strCurrentName}'", $OS_ERROR);
+ }
+
+ # Move the file from temp to final if atomic
+ if ($self->{bAtomic})
+ {
+ $self->{oDriver}->move($strCurrentName, $self->{strName});
+ }
+
+ # Set result
+ $self->resultSet(COMMON_IO_HANDLE, $self->{lSize});
+
+ # Close parent
+ $self->SUPER::close();
+ }
+
+ return true;
+}
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub opened {shift->{bOpened}}
+sub handle {shift->{fhFile}}
+
+1;
diff --git a/test/Vagrantfile b/test/Vagrantfile
index 2df2dba8e..4defe91ec 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -5,7 +5,10 @@ Vagrant.configure(2) do |config|
end
config.vm.box = "ubuntu/xenial64"
- config.vm.box_version = "20170311.0.0"
+ config.vm.box_version = "20170603.0.0"
+
+ # vagrant plugin install vagrant-disksize
+ # config.disksize.size = '64GB'
config.vm.provider :virtualbox do |vb|
vb.name = "pgbackrest-test"
diff --git a/test/expect/archive-get-001.log b/test/expect/archive-get-001.log
index 225bded4b..4d1607207 100644
--- a/test/expect/archive-get-001.log
+++ b/test/expect/archive-get-001.log
@@ -3,21 +3,19 @@ run 001 - rmt 0, cmp 0, exists 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
+P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed?
@@ -32,7 +30,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55
stanza-create db - create required data for stanza (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: stanza-create command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info
@@ -85,7 +83,8 @@ db-version="9.4"
stop all stanzas (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf stop
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo
+P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --repo-path=[TEST_PATH]/db-master/repo
+P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
@@ -95,7 +94,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000090000000900000009 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000090000000900000009
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000090000000900000009
P00 ERROR: [062]: stop file exists for all stanzas
@@ -109,7 +108,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 62
start all stanzas (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo
+P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --repo-path=[TEST_PATH]/db-master/repo
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
@@ -119,28 +118,24 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000090000000900000009 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000090000000900000009
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000090000000900000009
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000090000000900000009, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000090000000900000009, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000090000000900000009
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000090000000900000009-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000900000009, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = ()
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000090000000900000009
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000090000000900000009-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000900000009, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = ()
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = [undef]
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = [undef], strArchiveId = 9.4-1
P00 INFO: unable to find 000000090000000900000009 in the archive
diff --git a/test/expect/archive-get-002.log b/test/expect/archive-get-002.log
index 77048d950..6ae699c79 100644
--- a/test/expect/archive-get-002.log
+++ b/test/expect/archive-get-002.log
@@ -3,21 +3,19 @@ run 002 - rmt 0, cmp 0, exists 1
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
+P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed?
@@ -32,7 +30,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55
stanza-create db - create required data for stanza (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: stanza-create command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info
@@ -84,31 +82,30 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000100000001, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = (000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7)
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7)
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = false, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
@@ -119,31 +116,30 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000002
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strSourceArchive = 000000010000000100000002
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000002
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000100000001, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = (000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7)
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000002
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7)
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = false, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
@@ -154,31 +150,30 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000003 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000003
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strSourceArchive = 000000010000000100000003
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000003
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000003-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000100000001, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = (000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7)
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000003
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000003-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7)
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = false, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /9.4-1/000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
diff --git a/test/expect/archive-get-003.log b/test/expect/archive-get-003.log
index c73bec21a..be1ba6b82 100644
--- a/test/expect/archive-get-003.log
+++ b/test/expect/archive-get-003.log
@@ -3,21 +3,19 @@ run 003 - rmt 0, cmp 1, exists 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
+P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed?
@@ -32,7 +30,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55
stanza-create db - create required data for stanza (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: stanza-create command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info
@@ -85,7 +83,8 @@ db-version="9.4"
stop all stanzas (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf stop
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo
+P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --repo-path=[TEST_PATH]/db-master/repo
+P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
@@ -95,7 +94,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000090000000900000009 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000090000000900000009
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000090000000900000009
P00 ERROR: [062]: stop file exists for all stanzas
@@ -109,7 +108,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 62
start all stanzas (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo
+P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --repo-path=[TEST_PATH]/db-master/repo
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
@@ -119,28 +118,24 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000090000000900000009 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000090000000900000009
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000090000000900000009
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000090000000900000009, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000090000000900000009, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000090000000900000009
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000090000000900000009-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000900000009, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = ()
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000090000000900000009
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000090000000900000009-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000900000009, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = ()
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = [undef]
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = [undef], strArchiveId = 9.4-1
P00 INFO: unable to find 000000090000000900000009 in the archive
diff --git a/test/expect/archive-get-004.log b/test/expect/archive-get-004.log
index 99bd3253b..3d1905c98 100644
--- a/test/expect/archive-get-004.log
+++ b/test/expect/archive-get-004.log
@@ -3,21 +3,19 @@ run 004 - rmt 0, cmp 1, exists 1
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
+P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed?
@@ -32,7 +30,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55
stanza-create db - create required data for stanza (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: stanza-create command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info
@@ -84,31 +82,30 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000100000001, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = (000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz)
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz)
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = true, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
@@ -119,31 +116,30 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000002
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strSourceArchive = 000000010000000100000002
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000002
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000100000001, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = (000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz)
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000002
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz)
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = true, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
@@ -154,31 +150,30 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000003 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: get WAL segment 000000010000000100000003
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strSourceArchive = 000000010000000100000003
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/db-master/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 DEBUG: Archive::ArchiveInfo->new(): bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info
P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918
P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
-P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oFile = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000003
-P00 DEBUG: File->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000003-[0-f]{40}(\.gz){0,1}$, strPath = 9.4-1/0000000100000001, strPathType = backup:archive, strSortOrder =
-P00 DEBUG: File->list=>: stryFileList = (000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz)
+P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000003
+P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000003-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder =
+P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz)
P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = true, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /9.4-1/000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
diff --git a/test/expect/archive-get-005.log b/test/expect/archive-get-005.log
index 9b2b3ce72..c1f24e462 100644
--- a/test/expect/archive-get-005.log
+++ b/test/expect/archive-get-005.log
@@ -3,23 +3,22 @@ run 005 - rmt 1, cmp 0, exists 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol
-P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strRemoteType = backup, strUser = [USER-1]
-P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote', strId = backup, strName = remote, strRemoteType = backup
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 ERROR: [055]: raised on backup host: archive.info does not exist but is required to push/get WAL segments
+P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
+P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
+P00 ERROR: [055]: raised on 'backup remote' host: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed?
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
@@ -35,7 +34,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55
stanza-create db - create required data for stanza (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-2] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-2] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=[TEST_PATH]/backup/repo --stanza=db
P00 INFO: stanza-create command end: completed successfully
+ supplemental file: [TEST_PATH]/backup/repo/backup/db/backup.info
@@ -88,7 +87,8 @@ db-version="9.4"
stop all stanzas (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf stop
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stop command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo
+P00 INFO: stop command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log
+P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
@@ -98,7 +98,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000090000000900000009 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000090000000900000009
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000090000000900000009
P00 ERROR: [062]: stop file exists for all stanzas
@@ -112,7 +112,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 62
start all stanzas (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: start command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo
+P00 INFO: start command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
@@ -122,22 +122,21 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000090000000900000009 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000090000000900000009
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000090000000900000009
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol
-P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strRemoteType = backup, strUser = [USER-1]
-P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote', strId = backup, strName = remote, strRemoteType = backup
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000090000000900000009, ullDbSysId = [undef]
+P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000090000000900000009, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
+P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = [undef], strArchiveId = 9.4-1
P00 INFO: unable to find 000000090000000900000009 in the archive
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 1
diff --git a/test/expect/archive-get-006.log b/test/expect/archive-get-006.log
index 05ba2751e..08168bd30 100644
--- a/test/expect/archive-get-006.log
+++ b/test/expect/archive-get-006.log
@@ -3,23 +3,22 @@ run 006 - rmt 1, cmp 0, exists 1
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol
-P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strRemoteType = backup, strUser = [USER-1]
-P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote', strId = backup, strName = remote, strRemoteType = backup
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
-P00 ERROR: [055]: raised on backup host: archive.info does not exist but is required to push/get WAL segments
+P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
+P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
+P00 ERROR: [055]: raised on 'backup remote' host: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed?
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
@@ -35,7 +34,7 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55
stanza-create db - create required data for stanza (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-2] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-2] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=[TEST_PATH]/backup/repo --stanza=db
P00 INFO: stanza-create command end: completed successfully
+ supplemental file: [TEST_PATH]/backup/repo/backup/db/backup.info
@@ -87,24 +86,26 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --cmd-ssh=/usr/bin/ssh 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol
-P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote, strCommandSSH = /usr/bin/ssh, strHost = backup, strRemoteType = backup, strUser = [USER-1]
-P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = /usr/bin/ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote', strId = backup, strName = remote, strRemoteType = backup
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = /usr/bin/ssh, strHost = backup, strUser = [USER-1]
+P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = /usr/bin/ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
+P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = false, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
@@ -117,24 +118,26 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000010000000100000002
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strSourceArchive = 000000010000000100000002
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol
-P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strRemoteType = backup, strUser = [USER-1]
-P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote', strId = backup, strName = remote, strRemoteType = backup
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
+P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
+P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum = , bDestinationCompress = false, bDestinationPathCreate = , bIgnoreMissingSource = , bPathSync = , bSourceCompressed = false, bTempFile = , lModificationTime = [undef], rExtraParam = [undef], strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strDestinationPathType = db:absolute, strExtraFunction = [undef], strGroup = [undef], strMode = <0640>, strSourceFile = 9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strSourcePathType = backup:archive, strUser = [undef]
+P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7
+P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0
P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
@@ -147,24 +150,26 @@ P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000003 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
------------------------------------------------------------------------------------------------------------------------------------
-P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db
+P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
P00 INFO: get WAL segment 000000010000000100000003
P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strSourceArchive = 000000010000000100000003
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol
-P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strRemoteType = backup, strUser = [USER-1]
-P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1830, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --protocol-timeout=1830 --repo-path=[TEST_PATH]/backup/repo --stanza=db --type=backup remote', strId = backup, strName = remote, strRemoteType = backup
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
-P00 DEBUG: Archive::Archive->getCheck(): oFile = [object], strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
+P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
+P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
P00 DEBUG: Db->new(): iRemoteIdx = 1
-P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db
-P00 DEBUG: Protocol::Helper::protocolGet: create local protocol
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
-P00 DEBUG: File->new(): oProtocol = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strRepoPath = [TEST_PATH]/backup/repo, strStanza = db
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup
+P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7, strArchiveId = 9.4-1
-P00 DEBUG: File->copy(): bAppendChecksum =