You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2026-01-06 08:01:21 +03:00
097eb7ca41
* Added new feature to allow all pgBackRest operations to be stopped or started using the stop and start commands. This prevents any pgBackRest processes from running on a system where PostgreSQL is shutdown or the system needs to be quiesced for some reason. * Removed dependency on IO::String module. * Fixed an issue where an error could be returned after a backup or restore completely successfully. * Fixed an issue where a resume would fail if temp files were left in the root backup directory when the backup failed. This scenario was likely if the backup process got terminated during the copy phase. * Experimental support for PostgreSQL 9.5 beta1. This may break when the control version or WAL magic changes in future versions but will be updated in each pgBackRest release to keep pace. All regression tests pass except for --target-resume tests (this functionality has changed in 9.5) and there is no testing yet for .partial WAL segments.
2030 lines
64 KiB
Perl
2030 lines
64 KiB
Perl
####################################################################################################################################
|
|
# FILE MODULE
|
|
####################################################################################################################################
|
|
package BackRest::File;
|
|
|
|
use strict;
|
|
use warnings FATAL => qw(all);
|
|
use Carp qw(confess);
|
|
|
|
use Exporter qw(import);
|
|
our @EXPORT = qw();
|
|
use Digest::SHA;
|
|
use Fcntl qw(:mode :flock O_RDONLY O_WRONLY O_CREAT O_EXCL);
|
|
use File::Basename qw(dirname basename);
|
|
use File::Copy qw(cp);
|
|
use File::Path qw(make_path remove_tree);
|
|
use File::stat;
|
|
use IO::Handle;
|
|
use Scalar::Util qw(blessed);
|
|
|
|
use lib dirname($0) . '/../lib';
|
|
use BackRest::Common::Exception;
|
|
use BackRest::Common::Log;
|
|
use BackRest::Common::String;
|
|
use BackRest::Common::Wait;
|
|
use BackRest::Config::Config;
|
|
use BackRest::FileCommon;
|
|
|
|
####################################################################################################################################
|
|
# Operation constants
|
|
####################################################################################################################################
|
|
use constant OP_FILE => 'File';
|
|
|
|
use constant OP_FILE_CLONE => OP_FILE . '->clone';
|
|
use constant OP_FILE_COMPRESS => OP_FILE . '->compress';
|
|
use constant OP_FILE_COPY => OP_FILE . '->copy';
|
|
push @EXPORT, qw(OP_FILE_COPY);
|
|
use constant OP_FILE_COPY_IN => OP_FILE . '->copyIn';
|
|
push @EXPORT, qw(OP_FILE_COPY_IN);
|
|
use constant OP_FILE_COPY_OUT => OP_FILE . '->copyOut';
|
|
push @EXPORT, qw(OP_FILE_COPY_OUT);
|
|
use constant OP_FILE_DESTROY => OP_FILE . '->DESTROY';
|
|
use constant OP_FILE_EXISTS => OP_FILE . '->exists';
|
|
push @EXPORT, qw(OP_FILE_EXISTS);
|
|
use constant OP_FILE_HASH => OP_FILE . '->hash';
|
|
use constant OP_FILE_HASH_SIZE => OP_FILE . '->hashSize';
|
|
use constant OP_FILE_LINK_CREATE => OP_FILE . '->linkCreate';
|
|
use constant OP_FILE_LIST => OP_FILE . '->list';
|
|
push @EXPORT, qw(OP_FILE_LIST);
|
|
use constant OP_FILE_MANIFEST => OP_FILE . '->manifest';
|
|
push @EXPORT, qw(OP_FILE_MANIFEST);
|
|
use constant OP_FILE_MANIFEST_RECURSE => OP_FILE . '->manifestRecurse';
|
|
use constant OP_FILE_MOVE => OP_FILE . '->move';
|
|
use constant OP_FILE_NEW => OP_FILE . '->new';
|
|
use constant OP_FILE_OWNER => OP_FILE . '->owner';
|
|
use constant OP_FILE_PATH_CREATE => OP_FILE . '->pathCreate';
|
|
push @EXPORT, qw(OP_FILE_PATH_CREATE);
|
|
use constant OP_FILE_PATH_GET => OP_FILE . '->pathGet';
|
|
use constant OP_FILE_PATH_SYNC => OP_FILE . '->pathSync';
|
|
use constant OP_FILE_PATH_SYNC_STATIC => OP_FILE . '::filePathSync';
|
|
use constant OP_FILE_PATH_TYPE_GET => OP_FILE . '->pathTypeGet';
|
|
use constant OP_FILE_REMOVE => OP_FILE . '->remove';
|
|
use constant OP_FILE_STANZA => OP_FILE . '->stanza';
|
|
use constant OP_FILE_WAIT => OP_FILE . '->wait';
|
|
push @EXPORT, qw(OP_FILE_WAIT);
|
|
|
|
####################################################################################################################################
|
|
# COMMAND error constants [DEPRECATED - TO BE REPLACED BY CONSTANTS IN EXCEPTION.PM]
|
|
####################################################################################################################################
|
|
use constant COMMAND_ERR_FILE_MISSING => 1;
|
|
use constant COMMAND_ERR_FILE_READ => 2;
|
|
use constant COMMAND_ERR_FILE_MOVE => 3;
|
|
use constant COMMAND_ERR_FILE_TYPE => 4;
|
|
use constant COMMAND_ERR_LINK_READ => 5;
|
|
use constant COMMAND_ERR_PATH_MISSING => 6;
|
|
use constant COMMAND_ERR_PATH_CREATE => 7;
|
|
use constant COMMAND_ERR_PATH_READ => 8;
|
|
|
|
####################################################################################################################################
|
|
# PATH_GET constants
|
|
####################################################################################################################################
|
|
use constant PATH_ABSOLUTE => 'absolute';
|
|
push @EXPORT, qw(PATH_ABSOLUTE);
|
|
use constant PATH_DB => 'db';
|
|
push @EXPORT, qw(PATH_DB);
|
|
use constant PATH_DB_ABSOLUTE => 'db:absolute';
|
|
push @EXPORT, qw(PATH_DB_ABSOLUTE);
|
|
use constant PATH_BACKUP => 'backup';
|
|
push @EXPORT, qw(PATH_BACKUP);
|
|
use constant PATH_BACKUP_ABSOLUTE => 'backup:absolute';
|
|
push @EXPORT, qw(PATH_BACKUP_ABSOLUTE);
|
|
use constant PATH_BACKUP_CLUSTER => 'backup:cluster';
|
|
push @EXPORT, qw(PATH_BACKUP_CLUSTER);
|
|
use constant PATH_BACKUP_TMP => 'backup:tmp';
|
|
push @EXPORT, qw(PATH_BACKUP_TMP);
|
|
use constant PATH_BACKUP_ARCHIVE => 'backup:archive';
|
|
push @EXPORT, qw(PATH_BACKUP_ARCHIVE);
|
|
use constant PATH_BACKUP_ARCHIVE_OUT => 'backup:archive:out';
|
|
push @EXPORT, qw(PATH_BACKUP_ARCHIVE_OUT);
|
|
|
|
####################################################################################################################################
|
|
# STD pipe constants
|
|
####################################################################################################################################
|
|
use constant PIPE_STDIN => '<STDIN>';
|
|
push @EXPORT, qw(PIPE_STDIN);
|
|
use constant PIPE_STDOUT => '<STDOUT>';
|
|
push @EXPORT, qw(PIPE_STDOUT);
|
|
use constant PIPE_STDERR => '<STDERR>';
|
|
push @EXPORT, qw(PIPE_STDERR);
|
|
|
|
####################################################################################################################################
|
|
# new
|
|
####################################################################################################################################
|
|
sub new
|
|
{
|
|
my $class = shift;
|
|
|
|
# Create the class hash
|
|
my $self = {};
|
|
bless $self, $class;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
(
|
|
my $strOperation,
|
|
$self->{strStanza},
|
|
$self->{strBackupPath},
|
|
$self->{strRemote},
|
|
$self->{oProtocol},
|
|
$self->{strDefaultPathMode},
|
|
$self->{strDefaultFileMode},
|
|
$self->{iThreadIdx}
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_NEW, \@_,
|
|
{name => 'strStanza', required => false},
|
|
{name => 'strBackupPath'},
|
|
{name => 'strRemote', required => false},
|
|
{name => 'oProtocol'},
|
|
{name => 'strDefaultPathMode', default => '0750'},
|
|
{name => 'strDefaultFileMode', default => '0640'},
|
|
{name => 'iThreadIdx', required => false}
|
|
);
|
|
|
|
# Default compression extension to gz
|
|
$self->{strCompressExtension} = 'gz';
|
|
|
|
# Remote object must be set
|
|
if (!defined($self->{oProtocol}))
|
|
{
|
|
confess &log(ASSERT, 'oProtocol must be defined');
|
|
}
|
|
|
|
# If remote is defined check parameters and open session
|
|
if (defined($self->{strRemote}) && $self->{strRemote} ne NONE)
|
|
{
|
|
# Make sure remote is valid
|
|
if ($self->{strRemote} ne DB && $self->{strRemote} ne BACKUP)
|
|
{
|
|
confess &log(ASSERT, 'strRemote must be "' . DB . '" or "' . BACKUP .
|
|
"\", $self->{strRemote} was passed");
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => $self}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# DESTROY
|
|
####################################################################################################################################
|
|
sub DESTROY
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_DESTROY
|
|
);
|
|
|
|
if (defined($self->{oProtocol}))
|
|
{
|
|
$self->{oProtocol} = undef;
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# clone
|
|
####################################################################################################################################
|
|
sub clone
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$iThreadIdx
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_CLONE, \@_,
|
|
{name => 'iThreadidx', required => false}
|
|
);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => BackRest::File->new
|
|
(
|
|
$self->{strStanza},
|
|
$self->{strBackupPath},
|
|
$self->{strRemote},
|
|
$self->{oProtocol},
|
|
$self->{strDefaultPathMode},
|
|
$self->{strDefaultFileMode},
|
|
$iThreadIdx
|
|
)}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# stanza
|
|
####################################################################################################################################
|
|
sub stanza
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_STANZA
|
|
);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strStanza', $self->{strStanza}, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# pathTypeGet
|
|
####################################################################################################################################
|
|
sub pathTypeGet
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strType
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_PATH_TYPE_GET, \@_,
|
|
{name => 'strType', trace => true}
|
|
);
|
|
|
|
my $strPath;
|
|
|
|
# If absolute type
|
|
if ($strType eq PATH_ABSOLUTE)
|
|
{
|
|
$strPath = PATH_ABSOLUTE;
|
|
}
|
|
# If db type
|
|
elsif ($strType =~ /^db(\:.*){0,1}/)
|
|
{
|
|
$strPath = PATH_DB;
|
|
}
|
|
# Else if backup type
|
|
elsif ($strType =~ /^backup(\:.*){0,1}/)
|
|
{
|
|
$strPath = PATH_BACKUP;
|
|
}
|
|
# Else error when path type not recognized
|
|
else
|
|
{
|
|
confess &log(ASSERT, "no known path types in '${strType}'");
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strPath', value => $strPath, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# pathGet
|
|
# !!! Need ot tackle the return paths in this function
|
|
####################################################################################################################################
|
|
sub pathGet
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strType, # Base type of the path to get (PATH_DB_ABSOLUTE, PATH_BACKUP_TMP, etc)
|
|
$strFile, # File to append to the base path (can include a path as well)
|
|
$bTemp # Return the temp file for this path type - only some types have temp files
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_PATH_GET, \@_,
|
|
{name => 'strType', trace => true},
|
|
{name => 'strFile', required => false, trace => true},
|
|
{name => 'bTemp', default => false, trace => true}
|
|
);
|
|
|
|
# Make sure that any absolute path starts with /, otherwise it will actually be relative
|
|
my $bAbsolute = $strType =~ /.*absolute.*/;
|
|
|
|
if ($bAbsolute && $strFile !~ /^\/.*/)
|
|
{
|
|
confess &log(ASSERT, "absolute path ${strType}:${strFile} must start with /");
|
|
}
|
|
|
|
# Only allow temp files for PATH_BACKUP_ARCHIVE, PATH_BACKUP_ARCHIVE_OUT, PATH_BACKUP_TMP and any absolute path
|
|
if ($bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_ARCHIVE_OUT || $strType eq PATH_BACKUP_TMP ||
|
|
$bAbsolute))
|
|
{
|
|
confess &log(ASSERT, 'temp file not supported on path ' . $strType);
|
|
}
|
|
|
|
# Get absolute path
|
|
if ($bAbsolute)
|
|
{
|
|
if (defined($bTemp) && $bTemp)
|
|
{
|
|
return $strFile . '.backrest.tmp';
|
|
}
|
|
|
|
return $strFile;
|
|
}
|
|
|
|
# Make sure the base backup path is defined (since all other path types are backup)
|
|
if (!defined($self->{strBackupPath}))
|
|
{
|
|
confess &log(ASSERT, 'strBackupPath not defined');
|
|
}
|
|
|
|
# Get base backup path
|
|
if ($strType eq PATH_BACKUP)
|
|
{
|
|
return $self->{strBackupPath} . (defined($strFile) ? "/${strFile}" : '');
|
|
}
|
|
|
|
# Make sure the cluster is defined
|
|
if (!defined($self->{strStanza}))
|
|
{
|
|
confess &log(ASSERT, 'strStanza not defined');
|
|
}
|
|
|
|
# Get the backup tmp path
|
|
if ($strType eq PATH_BACKUP_TMP)
|
|
{
|
|
my $strTempPath = "$self->{strBackupPath}/temp/$self->{strStanza}.tmp";
|
|
|
|
if ($bTemp)
|
|
{
|
|
return "${strTempPath}/file.tmp" . (defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : '');
|
|
}
|
|
|
|
return "${strTempPath}" . (defined($strFile) ? "/${strFile}" : '');
|
|
}
|
|
|
|
# Get the backup archive path
|
|
if ($strType eq PATH_BACKUP_ARCHIVE_OUT || $strType eq PATH_BACKUP_ARCHIVE)
|
|
{
|
|
my $strArchivePath = "$self->{strBackupPath}/archive/$self->{strStanza}";
|
|
|
|
if (!defined($strFile))
|
|
{
|
|
return $strArchivePath;
|
|
}
|
|
|
|
if ($strType eq PATH_BACKUP_ARCHIVE)
|
|
{
|
|
my $strArchiveId = (split('/', $strFile))[0];
|
|
my $strArchiveFile = (split('/', $strFile))[1];
|
|
|
|
if (!defined($strArchiveFile))
|
|
{
|
|
return "${strArchivePath}/${strFile}";
|
|
}
|
|
|
|
my $strArchive = substr(basename($strArchiveFile), 0, 24);
|
|
|
|
if ($strArchive !~ /^([0-F]){24}$/)
|
|
{
|
|
return "${strArchivePath}/${strFile}";
|
|
}
|
|
|
|
$strArchivePath = "${strArchivePath}/${strArchiveId}/" . substr($strArchive, 0, 16) . "/${strArchiveFile}";
|
|
}
|
|
else
|
|
{
|
|
$strArchivePath = "${strArchivePath}/out" . (defined($strFile) ? '/' . $strFile : '');
|
|
}
|
|
|
|
if ($bTemp)
|
|
{
|
|
if (!defined($strFile))
|
|
{
|
|
confess &log(ASSERT, 'archive temp must have strFile defined');
|
|
}
|
|
|
|
$strArchivePath = "${strArchivePath}.tmp";
|
|
}
|
|
|
|
return $strArchivePath;
|
|
}
|
|
|
|
if ($strType eq PATH_BACKUP_CLUSTER)
|
|
{
|
|
return $self->{strBackupPath} . "/backup/$self->{strStanza}" . (defined($strFile) ? "/${strFile}" : '');
|
|
}
|
|
|
|
# Error when path type not recognized
|
|
confess &log(ASSERT, "no known path types in '${strType}'");
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# isRemote
|
|
#
|
|
# Determine whether the path type is remote
|
|
####################################################################################################################################
|
|
sub isRemote
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_CLONE, \@_,
|
|
{name => 'strPathType', trace => true}
|
|
);
|
|
|
|
my $bRemote = defined($self->{strRemote}) && $self->pathTypeGet($strPathType) eq $self->{strRemote};
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bRemote', value => $bRemote, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# linkCreate
|
|
####################################################################################################################################
|
|
sub linkCreate
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strSourcePathType,
|
|
$strSourceFile,
|
|
$strDestinationPathType,
|
|
$strDestinationFile,
|
|
$bHard,
|
|
$bRelative,
|
|
$bPathCreate
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_LINK_CREATE, \@_,
|
|
{name => 'strSourcePathType'},
|
|
{name => 'strSourceFile'},
|
|
{name => 'strDestinationPathType'},
|
|
{name => 'strDestinationFile'},
|
|
{name => 'bHard', default => false},
|
|
{name => 'bRelative', default => false},
|
|
{name => 'bPathCreate', default => true}
|
|
);
|
|
|
|
# Source and destination path types must be the same (e.g. both PATH_DB or both PATH_BACKUP, etc.)
|
|
if ($self->pathTypeGet($strSourcePathType) ne $self->pathTypeGet($strDestinationPathType))
|
|
{
|
|
confess &log(ASSERT, 'path types must be equal in link create');
|
|
}
|
|
|
|
# Generate source and destination files
|
|
my $strSource = $self->pathGet($strSourcePathType, $strSourceFile);
|
|
my $strDestination = $self->pathGet($strDestinationPathType, $strDestinationFile);
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strSourcePathType))
|
|
{
|
|
confess &log(ASSERT, 'remote operation not supported');
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
# If the destination path is backup and does not exist, create it
|
|
# !!! This should only happen when the link create errors
|
|
if ($bPathCreate && $self->pathTypeGet($strDestinationPathType) eq PATH_BACKUP)
|
|
{
|
|
$self->pathCreate(PATH_BACKUP_ABSOLUTE, dirname($strDestination));
|
|
}
|
|
|
|
unless (-e $strSource)
|
|
{
|
|
if (-e $strSource . ".$self->{strCompressExtension}")
|
|
{
|
|
$strSource .= ".$self->{strCompressExtension}";
|
|
$strDestination .= ".$self->{strCompressExtension}";
|
|
}
|
|
else
|
|
{
|
|
# Error when a hardlink will be created on a missing file
|
|
if ($bHard)
|
|
{
|
|
confess &log(ASSERT, "unable to find ${strSource}(.$self->{strCompressExtension}) for link");
|
|
}
|
|
}
|
|
}
|
|
|
|
# Generate relative path if requested
|
|
if ($bRelative)
|
|
{
|
|
my $iCommonLen = commonPrefix($strSource, $strDestination);
|
|
|
|
if ($iCommonLen != 0)
|
|
{
|
|
$strSource = ('../' x substr($strDestination, $iCommonLen) =~ tr/\///) . substr($strSource, $iCommonLen);
|
|
}
|
|
|
|
logDebugMisc
|
|
(
|
|
$strOperation, 'apply relative path',
|
|
{name => 'strSource', value => $strSource, trace => true}
|
|
);
|
|
}
|
|
|
|
if ($bHard)
|
|
{
|
|
link($strSource, $strDestination)
|
|
or confess &log(ERROR, "unable to create hardlink from ${strSource} to ${strDestination}");
|
|
}
|
|
else
|
|
{
|
|
symlink($strSource, $strDestination)
|
|
or confess &log(ERROR, "unable to create symlink from ${strSource} to ${strDestination}");
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# pathSync
|
|
#
|
|
# Sync a directory.
|
|
####################################################################################################################################
|
|
sub pathSync
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPath,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_PATH_SYNC, \@_,
|
|
{name => 'strPathType', trace => true},
|
|
{name => 'strPath', trace => true}
|
|
);
|
|
|
|
filePathSync($self->pathGet($strPathType, $strPath eq '.' ? undef : $strPath));
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# move
|
|
#
|
|
# Moves a file locally or remotely.
|
|
####################################################################################################################################
|
|
sub move
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strSourcePathType,
|
|
$strSourceFile,
|
|
$strDestinationPathType,
|
|
$strDestinationFile,
|
|
$bDestinationPathCreate
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_MOVE, \@_,
|
|
{name => 'strSourcePathType'},
|
|
{name => 'strSourceFile', required => false},
|
|
{name => 'strDestinationPathType'},
|
|
{name => 'strDestinationFile'},
|
|
{name => 'bDestinationPathCreate', default => false}
|
|
);
|
|
|
|
# Source and destination path types must be the same
|
|
if ($self->pathTypeGet($strSourcePathType) ne $self->pathTypeGet($strSourcePathType))
|
|
{
|
|
confess &log(ASSERT, 'source and destination path types must be equal');
|
|
}
|
|
|
|
# Set operation variables
|
|
my $strPathOpSource = $self->pathGet($strSourcePathType, $strSourceFile);
|
|
my $strPathOpDestination = $self->pathGet($strDestinationPathType, $strDestinationFile);
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strSourcePathType))
|
|
{
|
|
confess &log(ASSERT, 'remote operation not supported');
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
if (!rename($strPathOpSource, $strPathOpDestination))
|
|
{
|
|
if ($bDestinationPathCreate)
|
|
{
|
|
$self->pathCreate(PATH_ABSOLUTE, dirname($strPathOpDestination), undef, true);
|
|
}
|
|
|
|
if (!$bDestinationPathCreate || !rename($strPathOpSource, $strPathOpDestination))
|
|
{
|
|
my $strError = "unable to move file ${strPathOpSource} to ${strPathOpDestination}: " . $!;
|
|
my $iErrorCode = COMMAND_ERR_FILE_READ;
|
|
|
|
if (!$self->exists(PATH_ABSOLUTE, dirname($strPathOpDestination)))
|
|
{
|
|
$strError = dirname($strPathOpDestination) . " destination path does not exist";
|
|
$iErrorCode = COMMAND_ERR_FILE_MISSING;
|
|
}
|
|
|
|
if (!($bDestinationPathCreate && $iErrorCode == COMMAND_ERR_FILE_MISSING))
|
|
{
|
|
if ($strSourcePathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
}
|
|
}
|
|
|
|
$self->pathSync($strDestinationPathType, dirname($strDestinationFile));
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# compress
|
|
####################################################################################################################################
|
|
sub compress
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strFile
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_COMPRESS, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strFile'}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strPathOp = $self->pathGet($strPathType, $strFile);
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
confess &log(ASSERT, 'remote operation not supported');
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
# Use copy to compress the file
|
|
$self->copy($strPathType, $strFile, $strPathType, "${strFile}.gz", false, true);
|
|
|
|
# Remove the old file
|
|
unlink($strPathOp)
|
|
or die &log(ERROR, "unable to remove ${strPathOp}");
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# pathCreate
|
|
#
|
|
# Creates a path locally or remotely.
|
|
####################################################################################################################################
|
|
sub pathCreate
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPath,
|
|
$strMode,
|
|
$bIgnoreExists
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_PATH_CREATE, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strPath', required => false},
|
|
{name => 'strMode', default => '0750'},
|
|
{name => 'bIgnoreExists', default => false}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strPathOp = $self->pathGet($strPathType, $strPath);
|
|
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
# Build param hash
|
|
my %oParamHash;
|
|
|
|
$oParamHash{path} = ${strPathOp};
|
|
|
|
if (defined($strMode))
|
|
{
|
|
$oParamHash{mode} = ${strMode};
|
|
}
|
|
|
|
# Execute the command
|
|
$self->{oProtocol}->cmdExecute(OP_FILE_PATH_CREATE, \%oParamHash, false);
|
|
}
|
|
else
|
|
{
|
|
if (!($bIgnoreExists && $self->exists($strPathType, $strPath)))
|
|
{
|
|
# Attempt the create the directory
|
|
my $stryError;
|
|
|
|
if (defined($strMode))
|
|
{
|
|
make_path($strPathOp, {mode => oct($strMode), error => \$stryError});
|
|
}
|
|
else
|
|
{
|
|
make_path($strPathOp, {error => \$stryError});
|
|
}
|
|
|
|
if (@$stryError)
|
|
{
|
|
# Capture the error
|
|
my $strError = "${strPathOp} could not be created: " . $!;
|
|
|
|
# If running on command line the return directly
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, COMMAND_ERR_PATH_CREATE);
|
|
}
|
|
|
|
# Error the normal way
|
|
confess &log(ERROR, $strError); #, COMMAND_ERR_PATH_CREATE);
|
|
}
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# exists
|
|
#
|
|
# Checks for the existence of a file, but does not imply that the file is readable/writeable.
|
|
#
|
|
# Return: true if file exists, false otherwise
|
|
####################################################################################################################################
|
|
sub exists
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPath
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_EXISTS, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strPath', required => false}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strPathOp = $self->pathGet($strPathType, $strPath);
|
|
my $bExists = true;
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
# Build param hash
|
|
my %oParamHash;
|
|
|
|
$oParamHash{path} = $strPathOp;
|
|
|
|
# Execute the command
|
|
$bExists = $self->{oProtocol}->cmdExecute($strOperation, \%oParamHash, true) eq 'Y' ? true : false;
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
# Stat the file/path to determine if it exists
|
|
my $oStat = lstat($strPathOp);
|
|
|
|
# Evaluate error
|
|
if (!defined($oStat))
|
|
{
|
|
# If the error is not entry missing, then throw error
|
|
if (!$!{ENOENT})
|
|
{
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $!, COMMAND_ERR_FILE_READ);
|
|
}
|
|
else
|
|
{
|
|
confess &log(ERROR, $!); #, COMMAND_ERR_FILE_READ);
|
|
}
|
|
}
|
|
|
|
$bExists = false;
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bExists', value => $bExists}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# remove
|
|
####################################################################################################################################
|
|
sub remove
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPath,
|
|
$bTemp,
|
|
$bIgnoreMissing
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_REMOVE, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strPath'},
|
|
{name => 'bTemp', required => false},
|
|
{name => 'bIgnoreMissing', default => true}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strPathOp = $self->pathGet($strPathType, $strPath, $bTemp);
|
|
my $bRemoved = true;
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
confess &log(ASSERT, OP_FILE_REMOVE . ": remote operation not supported");
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
if (unlink($strPathOp) != 1)
|
|
{
|
|
$bRemoved = false;
|
|
|
|
my $strError = "${strPathOp} could not be removed: " . $!;
|
|
my $iErrorCode = COMMAND_ERR_PATH_READ;
|
|
|
|
if (!$self->exists($strPathType, $strPath))
|
|
{
|
|
$strError = "${strPathOp} does not exist";
|
|
$iErrorCode = COMMAND_ERR_PATH_MISSING;
|
|
}
|
|
|
|
if (!($iErrorCode == COMMAND_ERR_PATH_MISSING && $bIgnoreMissing))
|
|
{
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bRemoved', value => $bRemoved}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# hash
|
|
####################################################################################################################################
|
|
sub hash
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strFile,
|
|
$bCompressed,
|
|
$strHashType
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_HASH, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strFile'},
|
|
{name => 'bCompressed', required => false},
|
|
{name => 'strHashType', required => false}
|
|
);
|
|
|
|
my ($strHash) = $self->hashSize($strPathType, $strFile, $bCompressed, $strHashType);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strHash', value => $strHash, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# hashSize
|
|
####################################################################################################################################
|
|
sub hashSize
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strFile,
|
|
$bCompressed,
|
|
$strHashType
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_HASH_SIZE, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strFile'},
|
|
{name => 'bCompressed', default => false},
|
|
{name => 'strHashType', default => 'sha1'}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strFileOp = $self->pathGet($strPathType, $strFile);
|
|
my $strHash;
|
|
my $iSize = 0;
|
|
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
confess &log(ASSERT, OP_FILE_HASH_SIZE . ": remote operation not supported");
|
|
}
|
|
else
|
|
{
|
|
my $hFile;
|
|
|
|
if (!sysopen($hFile, $strFileOp, O_RDONLY))
|
|
{
|
|
my $strError = "${strFileOp} could not be read: " . $!;
|
|
my $iErrorCode = 2;
|
|
|
|
if (!$self->exists($strPathType, $strFile))
|
|
{
|
|
$strError = "${strFileOp} does not exist";
|
|
$iErrorCode = 1;
|
|
}
|
|
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
|
|
my $oSHA = Digest::SHA->new($strHashType);
|
|
|
|
if ($bCompressed)
|
|
{
|
|
($strHash, $iSize) =
|
|
$self->{oProtocol}->binaryXfer($hFile, 'none', 'in', true, false, false);
|
|
}
|
|
else
|
|
{
|
|
my $iBlockSize;
|
|
my $tBuffer;
|
|
|
|
do
|
|
{
|
|
# Read a block from the file
|
|
$iBlockSize = sysread($hFile, $tBuffer, 4194304);
|
|
|
|
if (!defined($iBlockSize))
|
|
{
|
|
confess &log(ERROR, "${strFileOp} could not be read: " . $!);
|
|
}
|
|
|
|
$iSize += $iBlockSize;
|
|
$oSHA->add($tBuffer);
|
|
}
|
|
while ($iBlockSize > 0);
|
|
|
|
$strHash = $oSHA->hexdigest();
|
|
}
|
|
|
|
close($hFile);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strHash', value => $strHash},
|
|
{name => 'iSize', value => $iSize}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# owner
|
|
####################################################################################################################################
|
|
sub owner
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strFile,
|
|
$strUser,
|
|
$strGroup
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_OWNER, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strFile'},
|
|
{name => 'strUser'},
|
|
{name => 'strGroup'}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strFileOp = $self->pathGet($strPathType, $strFile);
|
|
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
confess &log(ASSERT, OP_FILE_OWNER . ": remote operation not supported");
|
|
}
|
|
else
|
|
{
|
|
my $iUserId;
|
|
my $iGroupId;
|
|
my $oStat;
|
|
|
|
if (!defined($strUser) || !defined($strGroup))
|
|
{
|
|
$oStat = stat($strFileOp);
|
|
|
|
if (!defined($oStat))
|
|
{
|
|
confess &log(ERROR, 'unable to stat ${strFileOp}');
|
|
}
|
|
}
|
|
|
|
if (defined($strUser))
|
|
{
|
|
$iUserId = getpwnam($strUser);
|
|
}
|
|
else
|
|
{
|
|
$iUserId = $oStat->uid;
|
|
}
|
|
|
|
if (defined($strGroup))
|
|
{
|
|
$iGroupId = getgrnam($strGroup);
|
|
}
|
|
else
|
|
{
|
|
$iGroupId = $oStat->gid;
|
|
}
|
|
|
|
chown($iUserId, $iGroupId, $strFileOp)
|
|
or confess &log(ERROR, "unable to set ownership for ${strFileOp}");
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# list
|
|
####################################################################################################################################
|
|
sub list
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPath,
|
|
$strExpression,
|
|
$strSortOrder,
|
|
$bIgnoreMissing
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_LIST, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strPath', required => false},
|
|
{name => 'strExpression', required => false},
|
|
{name => 'strSortOrder', default => 'forward'},
|
|
{name => 'bIgnoreMissing', default => false}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strPathOp = $self->pathGet($strPathType, $strPath);
|
|
my @stryFileList;
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
# Build param hash
|
|
my %oParamHash;
|
|
|
|
$oParamHash{path} = $strPathOp;
|
|
$oParamHash{sort_order} = $strSortOrder;
|
|
$oParamHash{ignore_missing} = ${bIgnoreMissing};
|
|
|
|
if (defined($strExpression))
|
|
{
|
|
$oParamHash{expression} = $strExpression;
|
|
}
|
|
|
|
# Execute the command
|
|
my $strOutput = $self->{oProtocol}->cmdExecute(OP_FILE_LIST, \%oParamHash, false);
|
|
|
|
if (defined($strOutput))
|
|
{
|
|
@stryFileList = split(/\n/, $strOutput);
|
|
}
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
my $hPath;
|
|
|
|
if (!opendir($hPath, $strPathOp))
|
|
{
|
|
my $strError = "${strPathOp} could not be read: " . $!;
|
|
my $iErrorCode = COMMAND_ERR_PATH_READ;
|
|
|
|
if (!$self->exists($strPathType, $strPath))
|
|
{
|
|
# If ignore missing is set then return an empty array
|
|
if ($bIgnoreMissing)
|
|
{
|
|
return @stryFileList;
|
|
}
|
|
|
|
$strError = "${strPathOp} does not exist";
|
|
$iErrorCode = COMMAND_ERR_PATH_MISSING;
|
|
}
|
|
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
|
|
@stryFileList = grep(!/^(\.)|(\.\.)$/i, readdir($hPath));
|
|
|
|
close($hPath);
|
|
|
|
if (defined($strExpression))
|
|
{
|
|
@stryFileList = grep(/$strExpression/i, @stryFileList);
|
|
}
|
|
|
|
# Reverse sort
|
|
if ($strSortOrder eq 'reverse')
|
|
{
|
|
@stryFileList = sort {$b cmp $a} @stryFileList;
|
|
}
|
|
# Normal sort
|
|
else
|
|
{
|
|
@stryFileList = sort @stryFileList;
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'stryFileList', value => \@stryFileList}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# wait
|
|
#
|
|
# Wait until the next second. This is done in the file object because it must be performed on whichever side the db is on, local or
|
|
# remote. This function is used to make sure that no files are copied in the same second as the manifest is created. The reason is
|
|
# that the db might modify they file again in the same second as the copy and that change will not be visible to a subsequent
|
|
# incremental backup using timestamp/size to determine deltas.
|
|
####################################################################################################################################
|
|
sub wait
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_WAIT, \@_,
|
|
{name => 'strPathType'}
|
|
);
|
|
|
|
# Second when the function was called
|
|
my $lTimeBegin;
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
# Execute the command
|
|
$lTimeBegin = $self->{oProtocol}->cmdExecute(OP_FILE_WAIT, undef, true);
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
# Wait the remainder of the current second
|
|
$lTimeBegin = waitRemainder();
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'lTimeBegin', value => $lTimeBegin, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# manifest
|
|
#
|
|
# Builds a path/file manifest starting with the base path and including all subpaths. The manifest contains all the information
|
|
# needed to perform a backup or a delta with a previous backup.
|
|
####################################################################################################################################
|
|
sub manifest
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPath,
|
|
$oManifestHashRef
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_MANIFEST, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strPath', required => false},
|
|
{name => 'oManifestHashRef'}
|
|
);
|
|
|
|
# Set operation variables
|
|
my $strPathOp = $self->pathGet($strPathType, $strPath);
|
|
|
|
# Run remotely
|
|
if ($self->isRemote($strPathType))
|
|
{
|
|
# Build param hash
|
|
my %oParamHash;
|
|
|
|
$oParamHash{path} = $strPathOp;
|
|
|
|
# Execute the command
|
|
dataHashBuild($oManifestHashRef, $self->{oProtocol}->cmdExecute(OP_FILE_MANIFEST, \%oParamHash, true), "\t");
|
|
}
|
|
# Run locally
|
|
else
|
|
{
|
|
$self->manifestRecurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
sub manifestRecurse
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strPathType,
|
|
$strPathOp,
|
|
$strPathFileOp,
|
|
$iDepth,
|
|
$oManifestHashRef
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_MANIFEST_RECURSE, \@_,
|
|
{name => 'strPathType'},
|
|
{name => 'strPathOp'},
|
|
{name => 'strPathFileOp', required => false},
|
|
{name => 'iDepth'},
|
|
{name => 'oManifestHashRef', required => false}
|
|
);
|
|
|
|
# Set operation and debug strings
|
|
my $strPathRead = $strPathOp . (defined($strPathFileOp) ? "/${strPathFileOp}" : '');
|
|
my $hPath;
|
|
|
|
# Open the path
|
|
if (!opendir($hPath, $strPathRead))
|
|
{
|
|
my $strError = "${strPathRead} could not be read: " . $!;
|
|
my $iErrorCode = COMMAND_ERR_PATH_READ;
|
|
|
|
# If the path does not exist and is not the root path requested then return, else error
|
|
# It's OK for paths to go away during execution (databases are a dynamic thing!)
|
|
if (!$self->exists(PATH_ABSOLUTE, $strPathRead))
|
|
{
|
|
if ($iDepth != 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
$strError = "${strPathRead} does not exist";
|
|
$iErrorCode = COMMAND_ERR_PATH_MISSING;
|
|
}
|
|
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
|
|
# Get a list of all files in the path (except ..)
|
|
my @stryFileList = grep(!/^\..$/i, readdir($hPath));
|
|
|
|
close($hPath);
|
|
|
|
# Loop through all subpaths/files in the path
|
|
foreach my $strFile (sort(@stryFileList))
|
|
{
|
|
my $strPathFile = "${strPathRead}/$strFile";
|
|
my $bCurrentDir = $strFile eq '.';
|
|
|
|
# Create the file and path names
|
|
if ($iDepth != 0)
|
|
{
|
|
if ($bCurrentDir)
|
|
{
|
|
$strFile = $strPathFileOp;
|
|
$strPathFile = $strPathRead;
|
|
}
|
|
else
|
|
{
|
|
$strFile = "${strPathFileOp}/${strFile}";
|
|
}
|
|
}
|
|
|
|
# Stat the path/file
|
|
my $oStat = lstat($strPathFile);
|
|
|
|
# Check for errors in stat
|
|
if (!defined($oStat))
|
|
{
|
|
my $strError = "${strPathFile} could not be read: " . $!;
|
|
my $iErrorCode = COMMAND_ERR_FILE_READ;
|
|
|
|
# If the file does not exist then go to the next file, else error
|
|
# It's OK for files to go away during execution (databases are a dynamic thing!)
|
|
if (!$self->exists(PATH_ABSOLUTE, $strPathFile))
|
|
{
|
|
next;
|
|
}
|
|
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
|
|
# Check for regular file
|
|
if (S_ISREG($oStat->mode))
|
|
{
|
|
${$oManifestHashRef}{name}{"${strFile}"}{type} = 'f';
|
|
|
|
# Get inode
|
|
${$oManifestHashRef}{name}{"${strFile}"}{inode} = $oStat->ino;
|
|
|
|
# Get size
|
|
${$oManifestHashRef}{name}{"${strFile}"}{size} = $oStat->size;
|
|
|
|
# Get modification time
|
|
${$oManifestHashRef}{name}{"${strFile}"}{modification_time} = $oStat->mtime;
|
|
}
|
|
# Check for directory
|
|
elsif (S_ISDIR($oStat->mode))
|
|
{
|
|
${$oManifestHashRef}{name}{"${strFile}"}{type} = 'd';
|
|
}
|
|
# Check for link
|
|
elsif (S_ISLNK($oStat->mode))
|
|
{
|
|
${$oManifestHashRef}{name}{"${strFile}"}{type} = 'l';
|
|
|
|
# Get link destination
|
|
${$oManifestHashRef}{name}{"${strFile}"}{link_destination} = readlink($strPathFile);
|
|
|
|
if (!defined(${$oManifestHashRef}{name}{"${strFile}"}{link_destination}))
|
|
{
|
|
if (-e $strPathFile)
|
|
{
|
|
my $strError = "${strPathFile} error reading link: " . $!;
|
|
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
print $strError;
|
|
exit COMMAND_ERR_LINK_READ;
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
}
|
|
}
|
|
# Not a recognized type
|
|
else
|
|
{
|
|
my $strError = "${strPathFile} is not of type directory, file, or link";
|
|
|
|
if ($strPathType eq PATH_ABSOLUTE)
|
|
{
|
|
print $strError;
|
|
exit COMMAND_ERR_FILE_TYPE;
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
|
|
# Get user name
|
|
${$oManifestHashRef}{name}{"${strFile}"}{user} = getpwuid($oStat->uid);
|
|
|
|
# Get group name
|
|
${$oManifestHashRef}{name}{"${strFile}"}{group} = getgrgid($oStat->gid);
|
|
|
|
# Get mode
|
|
if (${$oManifestHashRef}{name}{"${strFile}"}{type} ne 'l')
|
|
{
|
|
${$oManifestHashRef}{name}{"${strFile}"}{mode} = sprintf('%04o', S_IMODE($oStat->mode));
|
|
}
|
|
|
|
# Recurse into directories
|
|
if (${$oManifestHashRef}{name}{"${strFile}"}{type} eq 'd' && !$bCurrentDir)
|
|
{
|
|
$self->manifestRecurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef);
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# copy
|
|
#
|
|
# Copies a file from one location to another:
|
|
#
|
|
# * source and destination can be local or remote
|
|
# * wire and output compression/decompression are supported
|
|
# * intermediate temp files are used to prevent partial copies
|
|
# * modification time, mode, and ownership can be set on destination file
|
|
# * destination path can optionally be created
|
|
####################################################################################################################################
|
|
sub copy
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strSourcePathType,
|
|
$strSourceFile,
|
|
$strDestinationPathType,
|
|
$strDestinationFile,
|
|
$bSourceCompressed,
|
|
$bDestinationCompress,
|
|
$bIgnoreMissingSource,
|
|
$lModificationTime,
|
|
$strMode,
|
|
$bDestinationPathCreate,
|
|
$strUser,
|
|
$strGroup,
|
|
$bAppendChecksum
|
|
) =
|
|
logDebugParam
|
|
(
|
|
OP_FILE_COPY, \@_,
|
|
{name => 'strSourcePathType'},
|
|
{name => 'strSourceFile', required => false},
|
|
{name => 'strDestinationPathType'},
|
|
{name => 'strDestinationFile', required => false},
|
|
{name => 'bSourceCompressed', default => false},
|
|
{name => 'bDestinationCompress', default => false},
|
|
{name => 'bIgnoreMissingSource', default => false},
|
|
{name => 'lModificationTime', required => false},
|
|
{name => 'strMode', default => '0640'},
|
|
{name => 'bDestinationPathCreate', default => false},
|
|
{name => 'strUser', required => false},
|
|
{name => 'strGroup', required => false},
|
|
{name => 'bAppendChecksum', default => false}
|
|
);
|
|
|
|
# Set working variables
|
|
my $bSourceRemote = $self->isRemote($strSourcePathType) || $strSourcePathType eq PIPE_STDIN;
|
|
my $bDestinationRemote = $self->isRemote($strDestinationPathType) || $strDestinationPathType eq PIPE_STDOUT;
|
|
my $strSourceOp = $strSourcePathType eq PIPE_STDIN ?
|
|
$strSourcePathType : $self->pathGet($strSourcePathType, $strSourceFile);
|
|
my $strDestinationOp = $strDestinationPathType eq PIPE_STDOUT ?
|
|
$strDestinationPathType : $self->pathGet($strDestinationPathType, $strDestinationFile);
|
|
my $strDestinationTmpOp = $strDestinationPathType eq PIPE_STDOUT ?
|
|
undef : $self->pathGet($strDestinationPathType, $strDestinationFile, true);
|
|
|
|
# Checksum and size variables
|
|
my $strChecksum = undef;
|
|
my $iFileSize = undef;
|
|
my $bResult = true;
|
|
|
|
# Open the source and destination files (if needed)
|
|
my $hSourceFile;
|
|
my $hDestinationFile;
|
|
|
|
if (!$bSourceRemote)
|
|
{
|
|
if (!sysopen($hSourceFile, $strSourceOp, O_RDONLY))
|
|
{
|
|
my $strError = $!;
|
|
my $iErrorCode = COMMAND_ERR_FILE_READ;
|
|
|
|
if ($!{ENOENT})
|
|
{
|
|
# $strError = 'file is missing';
|
|
$iErrorCode = COMMAND_ERR_FILE_MISSING;
|
|
|
|
if ($bIgnoreMissingSource && $strDestinationPathType ne PIPE_STDOUT)
|
|
{
|
|
return false, undef, undef;
|
|
}
|
|
}
|
|
|
|
$strError = "cannot open source file ${strSourceOp}: " . $strError;
|
|
|
|
if ($strSourcePathType eq PATH_ABSOLUTE)
|
|
{
|
|
if ($strDestinationPathType eq PIPE_STDOUT)
|
|
{
|
|
$self->{oProtocol}->binaryXferAbort();
|
|
}
|
|
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
}
|
|
|
|
if (!$bDestinationRemote)
|
|
{
|
|
my $iCreateFlag = O_WRONLY | O_CREAT | O_EXCL;
|
|
|
|
# Open the destination temp file
|
|
if (!sysopen($hDestinationFile, $strDestinationTmpOp, $iCreateFlag))
|
|
{
|
|
if ($bDestinationPathCreate)
|
|
{
|
|
$self->pathCreate(PATH_ABSOLUTE, dirname($strDestinationTmpOp), undef, true);
|
|
}
|
|
|
|
if (!$bDestinationPathCreate || !sysopen($hDestinationFile, $strDestinationTmpOp, $iCreateFlag))
|
|
{
|
|
my $strError = "unable to open ${strDestinationTmpOp}: " . $!;
|
|
my $iErrorCode = COMMAND_ERR_FILE_READ;
|
|
|
|
if (!$self->exists(PATH_ABSOLUTE, dirname($strDestinationTmpOp)))
|
|
{
|
|
$strError = dirname($strDestinationTmpOp) . ' destination path does not exist';
|
|
$iErrorCode = COMMAND_ERR_FILE_MISSING;
|
|
}
|
|
|
|
if (!($bDestinationPathCreate && $iErrorCode == COMMAND_ERR_FILE_MISSING))
|
|
{
|
|
if ($strSourcePathType eq PATH_ABSOLUTE)
|
|
{
|
|
confess &log(ERROR, $strError, $iErrorCode);
|
|
}
|
|
|
|
confess &log(ERROR, $strError);
|
|
}
|
|
}
|
|
}
|
|
|
|
# Now lock the file to be sure nobody else is operating on it
|
|
if (!flock($hDestinationFile, LOCK_EX | LOCK_NB))
|
|
{
|
|
confess &log(ERROR, "unable to acquire exclusive lock on lock ${strDestinationTmpOp}", ERROR_LOCK_ACQUIRE);
|
|
}
|
|
}
|
|
|
|
# If source or destination are remote
|
|
if ($bSourceRemote || $bDestinationRemote)
|
|
{
|
|
# Build the command and open the local file
|
|
my $hFile;
|
|
my %oParamHash;
|
|
my $hIn,
|
|
my $hOut;
|
|
my $strRemote;
|
|
my $strOperation;
|
|
|
|
# If source is remote and destination is local
|
|
if ($bSourceRemote && !$bDestinationRemote)
|
|
{
|
|
$hOut = $hDestinationFile;
|
|
$strOperation = OP_FILE_COPY_OUT;
|
|
$strRemote = 'in';
|
|
|
|
if ($strSourcePathType ne PIPE_STDIN)
|
|
{
|
|
$oParamHash{source_file} = $strSourceOp;
|
|
$oParamHash{source_compressed} = $bSourceCompressed;
|
|
$oParamHash{destination_compress} = $bDestinationCompress;
|
|
}
|
|
}
|
|
# Else if source is local and destination is remote
|
|
elsif (!$bSourceRemote && $bDestinationRemote)
|
|
{
|
|
$hIn = $hSourceFile;
|
|
$strOperation = OP_FILE_COPY_IN;
|
|
$strRemote = 'out';
|
|
|
|
if ($strDestinationPathType ne PIPE_STDOUT)
|
|
{
|
|
$oParamHash{destination_file} = $strDestinationOp;
|
|
$oParamHash{source_compressed} = $bSourceCompressed;
|
|
$oParamHash{destination_compress} = $bDestinationCompress;
|
|
$oParamHash{destination_path_create} = $bDestinationPathCreate;
|
|
|
|
if (defined($strMode))
|
|
{
|
|
$oParamHash{mode} = $strMode;
|
|
}
|
|
|
|
if (defined($strUser))
|
|
{
|
|
$oParamHash{user} = $strUser;
|
|
}
|
|
|
|
if (defined($strGroup))
|
|
{
|
|
$oParamHash{group} = $strGroup;
|
|
}
|
|
|
|
if ($bAppendChecksum)
|
|
{
|
|
$oParamHash{append_checksum} = true;
|
|
}
|
|
}
|
|
}
|
|
# Else source and destination are remote
|
|
else
|
|
{
|
|
$strOperation = OP_FILE_COPY;
|
|
|
|
$oParamHash{source_file} = $strSourceOp;
|
|
$oParamHash{source_compressed} = $bSourceCompressed;
|
|
$oParamHash{destination_file} = $strDestinationOp;
|
|
$oParamHash{destination_compress} = $bDestinationCompress;
|
|
$oParamHash{destination_path_create} = $bDestinationPathCreate;
|
|
|
|
if (defined($strMode))
|
|
{
|
|
$oParamHash{mode} = $strMode;
|
|
}
|
|
|
|
if (defined($strUser))
|
|
{
|
|
$oParamHash{user} = $strUser;
|
|
}
|
|
|
|
if (defined($strGroup))
|
|
{
|
|
$oParamHash{group} = $strGroup;
|
|
}
|
|
|
|
if ($bIgnoreMissingSource)
|
|
{
|
|
$oParamHash{ignore_missing_source} = $bIgnoreMissingSource;
|
|
}
|
|
|
|
if ($bAppendChecksum)
|
|
{
|
|
$oParamHash{append_checksum} = true;
|
|
}
|
|
}
|
|
|
|
# If an operation is defined then write it
|
|
if (%oParamHash)
|
|
{
|
|
$self->{oProtocol}->cmdWrite($strOperation, \%oParamHash);
|
|
}
|
|
|
|
# Transfer the file (skip this for copies where both sides are remote)
|
|
if ($strOperation ne OP_FILE_COPY)
|
|
{
|
|
($strChecksum, $iFileSize) =
|
|
$self->{oProtocol}->binaryXfer($hIn, $hOut, $strRemote, $bSourceCompressed, $bDestinationCompress);
|
|
}
|
|
|
|
# If this is the controlling process then wait for OK from remote
|
|
if (%oParamHash)
|
|
{
|
|
# Test for an error when reading output
|
|
my $strOutput;
|
|
|
|
eval
|
|
{
|
|
$strOutput = $self->{oProtocol}->outputRead(true, undef, true);
|
|
|
|
# Check the result of the remote call
|
|
if (substr($strOutput, 0, 1) eq 'Y')
|
|
{
|
|
# If the operation was purely remote, get checksum/size
|
|
if ($strOperation eq OP_FILE_COPY ||
|
|
$strOperation eq OP_FILE_COPY_IN && $bSourceCompressed && !$bDestinationCompress)
|
|
{
|
|
# Checksum shouldn't already be set
|
|
if (defined($strChecksum) || defined($iFileSize))
|
|
{
|
|
confess &log(ASSERT, "checksum and size are already defined, but shouldn't be");
|
|
}
|
|
|
|
# Parse output and check to make sure tokens are defined
|
|
my @stryToken = split(/ /, $strOutput);
|
|
|
|
if (!defined($stryToken[1]) || !defined($stryToken[2]) ||
|
|
$stryToken[1] eq '?' && $stryToken[2] eq '?')
|
|
{
|
|
confess &log(ERROR, "invalid return from copy" . (defined($strOutput) ? ": ${strOutput}" : ''));
|
|
}
|
|
|
|
# Read the checksum and size
|
|
if ($stryToken[1] ne '?')
|
|
{
|
|
$strChecksum = $stryToken[1];
|
|
}
|
|
|
|
if ($stryToken[2] ne '?')
|
|
{
|
|
$iFileSize = $stryToken[2];
|
|
}
|
|
}
|
|
}
|
|
# Remote called returned false
|
|
else
|
|
{
|
|
$bResult = false;
|
|
}
|
|
};
|
|
|
|
# If there is an error then evaluate
|
|
if ($@)
|
|
{
|
|
my $oMessage = $@;
|
|
|
|
# We'll ignore this error if the source file was missing and missing file exception was returned
|
|
# and bIgnoreMissingSource is set
|
|
if ($bIgnoreMissingSource && $strRemote eq 'in' && blessed($oMessage) && $oMessage->isa('BackRest::Common::Exception') &&
|
|
$oMessage->code() == COMMAND_ERR_FILE_MISSING)
|
|
{
|
|
close($hDestinationFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
|
|
unlink($strDestinationTmpOp) or confess &log(ERROR, "cannot remove file ${strDestinationTmpOp}");
|
|
|
|
return false, undef, undef;
|
|
}
|
|
|
|
confess $oMessage;
|
|
}
|
|
}
|
|
}
|
|
# Else this is a local operation
|
|
else
|
|
{
|
|
# If the source is not compressed and the destination is then compress
|
|
if (!$bSourceCompressed && $bDestinationCompress)
|
|
{
|
|
($strChecksum, $iFileSize) =
|
|
$self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'out', false, true, false);
|
|
}
|
|
# If the source is compressed and the destination is not then decompress
|
|
elsif ($bSourceCompressed && !$bDestinationCompress)
|
|
{
|
|
($strChecksum, $iFileSize) =
|
|
$self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'in', true, false, false);
|
|
}
|
|
# Else both side are compressed, so copy capturing checksum
|
|
elsif ($bSourceCompressed)
|
|
{
|
|
($strChecksum, $iFileSize) =
|
|
$self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'out', true, true, false);
|
|
}
|
|
else
|
|
{
|
|
($strChecksum, $iFileSize) =
|
|
$self->{oProtocol}->binaryXfer($hSourceFile, $hDestinationFile, 'in', false, true, false);
|
|
}
|
|
}
|
|
|
|
# Close the source file (if local)
|
|
if (defined($hSourceFile))
|
|
{
|
|
close($hSourceFile) or confess &log(ERROR, "cannot close file ${strSourceOp}");
|
|
}
|
|
|
|
# Sync and close the destination file (if local)
|
|
if (defined($hDestinationFile))
|
|
{
|
|
$hDestinationFile->sync()
|
|
or confess &log(ERROR, "unable to sync ${strDestinationTmpOp}", ERROR_FILE_SYNC);
|
|
|
|
close($hDestinationFile)
|
|
or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
|
|
}
|
|
|
|
# Checksum and file size should be set if the destination is not remote
|
|
if ($bResult &&
|
|
!(!$bSourceRemote && $bDestinationRemote && $bSourceCompressed) &&
|
|
(!defined($strChecksum) || !defined($iFileSize)))
|
|
{
|
|
confess &log(ASSERT, 'checksum or file size not set');
|
|
}
|
|
|
|
# Where the destination is local, set mode, modification time, and perform move to final location
|
|
if ($bResult && !$bDestinationRemote)
|
|
{
|
|
# Set the file Mode if required
|
|
if (defined($strMode))
|
|
{
|
|
chmod(oct($strMode), $strDestinationTmpOp)
|
|
or confess &log(ERROR, "unable to set mode for local ${strDestinationTmpOp}");
|
|
}
|
|
|
|
# Set the file modification time if required
|
|
if (defined($lModificationTime))
|
|
{
|
|
utime($lModificationTime, $lModificationTime, $strDestinationTmpOp)
|
|
or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
|
|
}
|
|
|
|
# set user and/or group if required
|
|
if (defined($strUser) || defined($strGroup))
|
|
{
|
|
$self->owner(PATH_ABSOLUTE, $strDestinationTmpOp, $strUser, $strGroup);
|
|
}
|
|
|
|
# Replace checksum in destination filename (if exists)
|
|
if ($bAppendChecksum)
|
|
{
|
|
# Replace destination filename
|
|
if ($bDestinationCompress)
|
|
{
|
|
$strDestinationOp =
|
|
substr($strDestinationOp, 0, length($strDestinationOp) - length($self->{strCompressExtension}) - 1) .
|
|
'-' . $strChecksum . '.' . $self->{strCompressExtension};
|
|
}
|
|
else
|
|
{
|
|
$strDestinationOp .= '-' . $strChecksum;
|
|
}
|
|
}
|
|
|
|
# Move the file from tmp to final destination
|
|
$self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, $bDestinationPathCreate);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bResult', value => $bResult, trace => true},
|
|
{name => 'strChecksum', value => $strChecksum, trace => true},
|
|
{name => 'iFileSize', value => $iFileSize, trace => true}
|
|
);
|
|
}
|
|
|
|
1;
|